Change:
Optimize the memory usage of goroutine stack for serving watch requests. According to the of handling a watch request, the stack memory is 0x3768, which is very close to 16 KiB and Golang runtime will double it to 32 KiB once exceeded. In fact, it does exceed if we print the traceback deeper and take the fact that extra stack memory reserved by Golang itself into consideration. A naive optimization is to run as much as preparation handlers in a separate goroutine and then move the actual serving to another goroutine, so the former routine can free the stack memory earlier and make it more memory efficient. Given the extensive use of watch requests, 32 KiB per watch could add up to a very high amount of memory overhead for . A typical watch request commonly lasts for 30s or longer, which means this memory will be active for quite long. I see a great improvement in memory efficiency to do this optimization.
/cc
/sig api-machinery
/cc if you would like to take a look when you have time :) Thank you /triage accepted

Discussion:
genericfeatures.APIServerTracing: {Default: true, PreRelease: featuregate.Beta}, <ins> genericfeatures.APIServingWithRoutine: {Default: true, PreRelease: featuregate.Beta}, </ins> genericfeatures.ConsistentListFromCache: {Default: false, PreRelease: featuregate.Alpha}, genericfeatures.CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31

"k8s.io/apiserver/pkg/endpoints/request" "k8s.io/apiserver/pkg/features" "k8s.io/apiserver/pkg/registry/rest" <ins> genericfilters "k8s.io/apiserver/pkg/server/filters" </ins> utilfeature "k8s.io/apiserver/pkg/util/feature" "k8s.io/component-base/tracing" "k8s.io/klog/v2"

} klog.V(3).InfoS("Starting watch", "path", req.URL.Path, "resourceVersion", opts.ResourceVersion, "labels", opts.LabelSelector, "fields", opts.FieldSelector, "timeout", timeout) ctx, cancel := context.WithTimeout(ctx, timeout) <del> defer cancel() </del> <ins> defer func() { cancel() }() </ins> watcher, err := rw.Watch(ctx, &opts) if err != nil { scope.err(err, w, req)

scope.err(err, w, req) return } <del> requestInfo, _ := request.RequestInfoFrom(ctx) metrics.RecordLongRunning(req, requestInfo, metrics.APIServerComponent, func() { defer watcher.Stop() handler.ServeHTTP(w, req) }) </del> <ins> // Invalidate cancel() to defer until serve() is complete. deferredCancel := cancel cancel = func() {} serve := func() { defer deferredCancel() requestInfo, _ := request.RequestInfoFrom(ctx) metrics.RecordLongRunning(req, requestInfo, metrics.APIServerComponent, func() { defer watcher.Stop() handler.ServeHTTP(w, req) }) } // Run watch serving in a separate goroutine to allow freeing current stack memory t := genericfilters.TaskFrom(req.Context()) if t != nil { t.Func = serve } else { serve() } </ins> return }

// Add support for distributed tracing in the API Server APIServerTracing featuregate.Feature = "APIServerTracing" <ins> // owner: @linxiulei // beta: v1.30 // // Enables serving watch requests in separate goroutines. APIServingWithRoutine featuregate.Feature = "APIServingWithRoutine" </ins> // owner: @cici37 @jpbetz // kep: http://kep.k8s.io/3488 // alpha: v1.26

APIServerTracing: {Default: true, PreRelease: featuregate.Beta}, <ins> APIServingWithRoutine: {Default: true, PreRelease: featuregate.Beta}, </ins> ValidatingAdmissionPolicy: {Default: false, PreRelease: featuregate.Beta}, CustomResourceValidationExpressions: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // remove in 1.31

handler = genericapifilters.WithTracing(handler, c.TracerProvider) } handler = genericapifilters.WithLatencyTrackers(handler) <ins> // WithRoutine will execute future handlers in a separate goroutine and serving // handler in current goroutine to minimize the stack memory usage. It must be // after WithPanicRecover() to be protected from panics. if utilfeature.DefaultFeatureGate.Enabled(genericfeatures.APIServingWithRoutine) { handler = genericfilters.WithRoutine(handler, c.LongRunningFunc) } </ins> handler = genericapifilters.WithRequestInfo(handler, c.RequestInfoResolver) handler = genericapifilters.WithRequestReceivedTimestamp(handler) handler = genericapifilters.WithMuxAndDiscoveryComplete(handler, c.lifecycleSignals.MuxAndDiscoveryComplete.Signaled())

<ins> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package filters import ( "context" "net/http" "k8s.io/apiserver/pkg/endpoints/request" ) type taskKeyType int const taskKey taskKeyType = iota type Task struct { Func func() } func WithTask(parent context.Context, t *Task) context.Context { return request.WithValue(parent, taskKey, t) } func TaskFrom(ctx context.Context) *Task { t, _ := ctx.Value(taskKey).(*Task) return t } // WithRoutine returns an http.Handler that executes preparation of long running requests (i.e. watches) // in a separate Goroutine and then serves the long running request in the main Goroutine. Doing so allows // freeing stack memory used in preparation Goroutine for better memory efficiency. func WithRoutine(handler http.Handler, longRunning request.LongRunningRequestCheck) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { ctx := req.Context() requestInfo, _ := request.RequestInfoFrom(ctx) if !longRunning(req, requestInfo) { handler.ServeHTTP(w, req) return } req = req.WithContext(WithTask(ctx, &Task{})) panicCh := make(chan any, 1) go func() { defer func() { if r := recover(); r != nil { panicCh <- r } close(panicCh) }() handler.ServeHTTP(w, req) }() if p, ok := <-panicCh; ok { panic(p) } ctx = req.Context() if t := TaskFrom(ctx); t != nil && t.Func != nil { t.Func() } }) } </ins>

<ins> /* Copyright 2023 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package filters import ( "bytes" "net/http" "net/http/httptest" "strings" "testing" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/klog/v2" ) func TestPropogatingPanic(t *testing.T) { var buf bytes.Buffer klog.SetOutput(&buf) klog.LogToStderr(false) defer klog.LogToStderr(true) panicMsg := "panic as designed" handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { panic(panicMsg) }) resolver := &request.RequestInfoFactory{ APIPrefixes:          sets.NewString("api", "apis"), GrouplessAPIPrefixes: sets.NewString("api"), } ts := httptest.NewServer(WithRoutine(WithPanicRecovery(handler, resolver), func(_ *http.Request, _ *request.RequestInfo) bool { return true })) defer ts.Close() _, err := http.Get(ts.URL) if err == nil { t.Error("expected to receive an error") } klog.Flush() klog.SetOutput(&bytes.Buffer{}) // prevent further writes into buf capturedOutput := buf.String() if !strings.Contains(capturedOutput, panicMsg) || !strings.Contains(capturedOutput, "apiserver panic'd") { t.Errorf("unexpected out captured actual = %v", capturedOutput) } } func TestExecutionWithRoutine(t *testing.T) { var executed bool handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { t := TaskFrom(r.Context()) t.Func = func() { executed = true } }) ts := httptest.NewServer(WithRoutine(handler, func(_ *http.Request, _ *request.RequestInfo) bool { return true })) defer ts.Close() _, err := http.Get(ts.URL) if err != nil { t.Errorf("got unexpected error on request: %v", err) } if !executed { t.Error("expected to execute") } } </ins>