Change:
What happened: When the kube-apiserver timeouts on a request, we end up doubling the counts measured in : apiserverrequestcounts apiserverrequesttotal apiserverrequestlatencies apiserverrequestlatenciessummary apiserverrequestdurationseconds apiserverresponsesizes What you expected to happen: We only count our requests a single time in our metrics. How to reproduce it (as minimally and precisely as possible): Add a 60 second sleep in the func . You may want to filter by resource (like pods or something) and verb for testing (otherwise this will be quite annoying). Manually hit that endpoint and look at the metric output. /assign
/sig api-machinery instrumentation /cc
Thanks for filing. Can we, while fixing this, add a metric counting timed-out requests? You can't tell from the latency distribution, even if we made the buckets more useful.
Yeah, that's the current plan. I've relayed the idea to offline and he has concurred.

Discussion:
}, []string{"group", "version", "kind"}, ) <del> // Because of volatality of the base metric this is pre-aggregated one. Instead of reporing current usage all the time </del> <ins> // Because of volatility of the base metric this is pre-aggregated one. Instead of reporting current usage all the time </ins> // it reports maximal usage during the last second. currentInflightRequests = compbasemetrics.NewGaugeVec( &compbasemetrics.GaugeOpts{

}, []string{"requestKind"}, ) <ins> requestTerminationsTotal = compbasemetrics.NewCounterVec( &compbasemetrics.CounterOpts{ Name:           "apiserver_request_terminations_total", Help:           "Number of requests which apiserver terminated in self-defense.", StabilityLevel: compbasemetrics.ALPHA, }, []string{"verb", "group", "version", "resource", "subresource", "scope", "component", "code"}, ) </ins> kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl.exe))`) metrics = []resettableCollector{

WatchEvents, WatchEventsSizes, currentInflightRequests, <ins> requestTerminationsTotal, </ins> } )

currentInflightRequests.WithLabelValues(MutatingKind).Set(float64(mutating)) } <del> // Record records a single request to the standard metrics endpoints. For use by handlers that perform their own // processing. All API paths should use InstrumentRouteFunc implicitly. Use this instead of MonitorRequest if // you already have a RequestInfo object. func Record(req *http.Request, requestInfo *request.RequestInfo, component, contentType string, code int, responseSizeInBytes int, elapsed time.Duration) { </del> <ins> // RecordRequestTermination records that the request was terminated early as part of a resource // preservation or apiserver self-defense mechanism (e.g. timeouts, maxinflight throttling, // proxyHandler errors). RecordRequestTermination should only be called zero or one times // per request. func RecordRequestTermination(req *http.Request, requestInfo *request.RequestInfo, component string, code int) { </ins> if requestInfo == nil { requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path} }

// However, we need to tweak it e.g. to differentiate GET from LIST. verb := canonicalVerb(strings.ToUpper(req.Method), scope) if requestInfo.IsResourceRequest { <del> MonitorRequest(req, verb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, contentType, code, responseSizeInBytes, elapsed) </del> <ins> requestTerminationsTotal.WithLabelValues(cleanVerb(verb, req), requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(code)).Inc() </ins> } else { <del> MonitorRequest(req, verb, "", "", "", requestInfo.Path, scope, component, contentType, code, responseSizeInBytes, elapsed) </del> <ins> requestTerminationsTotal.WithLabelValues(cleanVerb(verb, req), "", "", "", requestInfo.Path, scope, component, codeToString(code)).Inc() </ins> } }

} } } <del> metrics.Record(r, requestInfo, metrics.APIServerComponent, "", http.StatusTooManyRequests, 0, 0) </del> <ins> metrics.RecordRequestTermination(r, requestInfo, metrics.APIServerComponent, http.StatusTooManyRequests) </ins> tooManyRequests(r, w) } }

postTimeoutFn := func() { cancel() <del> metrics.Record(req, requestInfo, metrics.APIServerComponent, "", http.StatusGatewayTimeout, 0, 0) </del> <ins> metrics.RecordRequestTermination(req, requestInfo, metrics.APIServerComponent, http.StatusGatewayTimeout) </ins> } return req, time.After(timeout), postTimeoutFn, apierrors.NewTimeoutError(fmt.Sprintf("request did not complete within %s", timeout), 0) }

return } // TODO: record long-running request differently? The long-running check func does not necessarily match the one of the aggregated apiserver <del> endpointmetrics.Record(req, info, aggregatorComponent, "", code, 0, 0) </del> <ins> endpointmetrics.RecordRequestTermination(req, info, aggregatorComponent, code) </ins> } func (r *proxyHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {