Change:
pull-kubernetes-unit TestPriorityAndFairnessWithPanicRecoveryAndTimeoutFilter 7/14 No response /sig api-machinery
/priority important-soon
/cc /triage accepted
and can this be related to ?
I see no direct relationship, but hey, anything's possible. We learned that with the misery in integration test failures when we merged not-really-guilty other stuff.
The expanded logging turned up some new evidence. See , where the complaint is now
The problem here is not APF, it is something in the timeout filter or the test server.
is something in APF triggering a timeout?
The flake definitely came in with on the prior commit on master: on the merge commit of
/assign
Note that the flaking test is deliberately suffering server-side timeouts. The symptom in flakes is that the test is checking that the client gets kube's designed response to a timeout while handling, a particular form HTTP response message, but sometimes the client gets something else (an http2 internal error?). is a change in metrics, and should have nothing to do with how timeouts while handling and detected or handled. I will see what I can see...
, regarding your comment above (): note that the output shown (the panic "as designed") is normal; if there are failures involved, the failing part was elided.
Also, the failures seem to be coming from one particular subtest () and that one does not involve designed panics.
On closer examination of the timeout filter, it appears that this "INTERNAL ERROR" is something that can happen normally in the course of handling a timeout. So I will adjust the test to accept this.
I have updated to include the fix for this flake.

Discussion:
func TestPriorityAndFairnessWithPanicRecoveryAndTimeoutFilter(t *testing.T) { epmetrics.Register() fcmetrics.Register() <ins> timeFmt := "15:04:05.999" </ins> t.Run("priority level concurrency is set to 1, request handler panics, next request should not be rejected", func(t *testing.T) { const (

server, requestGetter := newHTTP2ServerWithClient(handler, requestTimeout*2) defer server.Close() <del> // we send two requests, each with a client timeout of requestTimeout*2 seconds // this ensures the test does not block indefinitely if the server does not respond. //  - first request (expected to timeout as designed) sent from a new goroutine //  - second request (expected to be enqueued) is sent from a new goroutine </del> <ins> // This test involves two requests sent to the same priority level, which has 1 queue and // a concurrency limit of 1.  The handler chain include the timeout filter. // Each request is sent from a separate goroutine, with a client-side timeout that is // double the timeout filter's limit. // The first request should get dispatched immediately; execution (a) starts with closing // the channel that triggers the second client goroutine to send its request and then (b) // waits for both client goroutines to have gotten a response (expected to be timeouts). // The second request sits in the queue until the timeout filter does its thing, which // it does concurrently to both requests.  For the first request this should make the client // get a timeout response without directly affecting execution.  For the second request, the // fact that the timeout filter closes the request's Context.Done() causes the request to be // promptly ejected from its queue.  The goroutine doing the APF handling writes an HTTP // response message with status 429. // The timeout handler invokes its inner handler in one goroutine while reacting to the // passage of time in its original goroutine.  That reaction to a time out consists of either // (a) writing an HTTP response message with status 504 to indicate the timeout or (b) doing an // HTTP/2 stream close; the latter is done if either the connection has been "hijacked" or some // other goroutine (e.g., the one running the inner handler) has started to write a response. // In the scenario tested here, there is thus a race between two goroutines to respond to // the second request and any of their responses is allowed by the test. </ins> firstReqResultCh, secondReqResultCh := make(chan result, 1), make(chan result, 1) go func() { defer close(firstReqRoundTripDoneCh) <del> t.Logf("Sending request: %q", firstRequestTimesOutPath) </del> <ins> t.Logf("At %s, Sending request: %q", time.Now().Format(timeFmt), firstRequestTimesOutPath) </ins> resp, err := requestGetter(firstRequestTimesOutPath) <del> t.Logf("RoundTrip of request: %q has completed", firstRequestTimesOutPath) </del> <ins> t.Logf("At %s, RoundTrip of request: %q has completed", time.Now().Format(timeFmt), firstRequestTimesOutPath) </ins> firstReqResultCh <- result{err: err, response: resp} }() go func() {

defer close(secondReqRoundTripDoneCh) <-firstReqInProgressCh <del> t.Logf("Sending request: %q", secondRequestEnqueuedPath) </del> <ins> t.Logf("At %s, Sending request: %q", time.Now().Format(timeFmt), secondRequestEnqueuedPath) </ins> resp, err := requestGetter(secondRequestEnqueuedPath) <del> t.Logf("RoundTrip of request: %q has completed", secondRequestEnqueuedPath) </del> <ins> t.Logf("At %s, RoundTrip of request: %q has completed", time.Now().Format(timeFmt), secondRequestEnqueuedPath) </ins> secondReqResultCh <- result{err: err, response: resp} }() firstReqResult := <-firstReqResultCh if isClientTimeout(firstReqResult.err) { <del> t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", firstRequestTimesOutPath, firstReqResult.err.Error()) </del> <ins> t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", firstRequestTimesOutPath, fmtError(firstReqResult.err)) </ins> } t.Logf("Waiting for the inner handler of the request: %q to complete", firstRequestTimesOutPath) <-firstReqHandlerCompletedCh // first request is expected to time out. if firstRequestInnerHandlerWriteErr != http.ErrHandlerTimeout { <del> t.Fatalf("Expected error: %#v, but got: %#v", http.ErrHandlerTimeout, firstRequestInnerHandlerWriteErr) } if firstReqResult.err != nil { t.Fatalf("Expected request: %q to get a response, but got error: %#v", firstRequestTimesOutPath, firstReqResult.err) </del> <ins> t.Fatalf("Expected error: %#v, but got: %s", http.ErrHandlerTimeout, fmtError(firstRequestInnerHandlerWriteErr)) </ins> } <del> if firstReqResult.response.StatusCode != http.StatusGatewayTimeout { t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#v", http.StatusGatewayTimeout, firstRequestTimesOutPath, firstReqResult.response) </del> <ins> if isStreamReset(firstReqResult.err) || firstReqResult.response.StatusCode != http.StatusGatewayTimeout { // got what was expected } else if firstReqResult.err != nil { t.Fatalf("Expected request: %q to get a response or stream reset, but got error: %s", firstRequestTimesOutPath, fmtError(firstReqResult.err)) } else if firstReqResult.response.StatusCode != http.StatusGatewayTimeout { t.Errorf("Expected HTTP status code: %d for request: %q, but got: %#+v", http.StatusGatewayTimeout, firstRequestTimesOutPath, firstReqResult.response) </ins> } // second request is expected to either be rejected (ideal behavior) or time out (current approximation of the ideal behavior) secondReqResult := <-secondReqResultCh if isClientTimeout(secondReqResult.err) { <del> t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", secondRequestEnqueuedPath, secondReqResult.err.Error()) </del> <ins> t.Fatalf("the client has unexpectedly timed out - request: %q error: %s", secondRequestEnqueuedPath, fmtError(secondReqResult.err)) </ins> } if secondRequestExecuted { t.Errorf("Expected second request to be enqueued: %q", secondRequestEnqueuedPath) } <del> if secondReqResult.err != nil { t.Fatalf("Expected request: %q to get a response, but got error: %#v", secondRequestEnqueuedPath, secondReqResult.err) } if !(secondReqResult.response.StatusCode == http.StatusTooManyRequests || secondReqResult.response.StatusCode == http.StatusGatewayTimeout) { t.Errorf("Expected HTTP status code: %d or %d for request: %q, but got: %#v", http.StatusTooManyRequests, http.StatusGatewayTimeout, secondRequestEnqueuedPath, secondReqResult.response) </del> <ins> if isStreamReset(secondReqResult.err) || secondReqResult.response.StatusCode == http.StatusTooManyRequests || secondReqResult.response.StatusCode == http.StatusGatewayTimeout { // got what was expected } else if secondReqResult.err != nil { t.Fatalf("Expected request: %q to get a response or stream reset, but got error: %s", secondRequestEnqueuedPath, fmtError(secondReqResult.err)) } else if !(secondReqResult.response.StatusCode == http.StatusTooManyRequests || secondReqResult.response.StatusCode == http.StatusGatewayTimeout) { t.Errorf("Expected HTTP status code: %d or %d for request: %q, but got: %#+v", http.StatusTooManyRequests, http.StatusGatewayTimeout, secondRequestEnqueuedPath, secondReqResult.response) </ins> } close(stopCh)

controllerErr := <-controllerCompletedCh if controllerErr != nil { <del> t.Errorf("Expected no error from the controller, but got: %#v", controllerErr) </del> <ins> t.Errorf("Expected no error from the controller, but got: %s", fmtError(controllerErr)) </ins> } }) } <ins> func fmtError(err error) string { return fmt.Sprintf("%#+v=%q", err, err.Error()) } </ins> func startAPFController(t *testing.T, stopCh <-chan struct{}, apfConfiguration []runtime.Object, serverConcurrency int, requestWaitLimit time.Duration, plName string, plConcurrency int) (utilflowcontrol.Interface, <-chan error) { clientset := newClientset(t, apfConfiguration...)

if err == nil { t.Fatalf("expected the server to send an error, but got nil") } <del> uerr, ok := err.(*url.Error) if !ok { t.Fatalf("expected the error to be of type *url.Error, but got: %T", err) } if !strings.Contains(uerr.Error(), "INTERNAL_ERROR") { t.Fatalf("expected a stream reset error, but got: %s", uerr.Error()) </del> <ins> if isStreamReset(err) { return </ins> } <ins> t.Fatalf("expected a stream reset error, but got %#+v=%s", err, err.Error()) </ins> } func newClientset(t *testing.T, objects ...runtime.Object) clientset.Interface {

} return false } <ins> func isStreamReset(err error) bool { if err == nil { return false } if urlErr, ok := err.(*url.Error); ok { // Sadly, the client does not receive a more specific indication // of stream reset. return strings.Contains(urlErr.Err.Error(), "INTERNAL_ERROR") } return false } </ins>