Code change:
http2Options := &http2.Server{} if err := http2.ConfigureServer(s.Config, http2Options); err != nil { <del> t.Fatalf("failed to configure test server to be HTTP2 server, err: %v", err) </del> <ins> return nil, fmt.Errorf("failed to configure test server to be HTTP2 server, err: %v", err) </ins> } s.TLS = s.Config.TLSConfig <del> s.StartTLS() defer s.Close() </del> <ins> return s, nil } // watchResponse wraps watch response with data which server send and an error may occur. type watchResponse struct { // body is the response data which test GOAWAY server sent to client body []byte // err will be set to be a non-nil value if watch request is not end with EOF nor http2.GoAwayError err error } // newGOAWAYClient return a configured http client which used to request test GOAWAY server, a dial may specified // to encounter the TCP connection. func newGOAWAYClient(dial func(network, addr string, cfg *tls.Config) (net.Conn, error)) (*http.Client, error) { </ins> tlsConfig := &tls.Config{ InsecureSkipVerify: true, NextProtos:         []string{http2.NextProtoTLS}, } <ins> tr := &http.Transport{ Proxy:               http.ProxyFromEnvironment, TLSHandshakeTimeout: 10 * time.Second, TLSClientConfig:     tlsConfig, MaxIdleConnsPerHost: 25, DialTLSContext: func(ctx context.Context, network, addr string) (net.Conn, error) { if dial != nil { return dial(network, addr, tlsConfig) } return tls.Dial(network, addr, tlsConfig) }, } if err := http2.ConfigureTransport(tr); err != nil { return nil, err } client := &http.Client{ Transport: tr, } return client, nil } // requestGOAWAYServer request test GOAWAY server using specified method and data according to the given url. // A non-nil channel will be returned if the request is watch, and a watchResponse can be got from the channel when watch done. func requestGOAWAYServer(client *http.Client, serverBaseURL, url string) (<-chan watchResponse, error) { method := http.MethodGet var reqBody io.Reader if url == urlPost || url == urlPostWithGoaway { method = http.MethodPost reqBody = bytes.NewReader(requestPostBody) } req, err := http.NewRequest(method, serverBaseURL+url, reqBody) if err != nil { return nil, fmt.Errorf("unexpect new request error: %v", err) } resp, err := client.Do(req) if err != nil { return nil, fmt.Errorf("failed request test server, err: %v", err) } if resp.StatusCode != http.StatusOK { defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body and status code is %d, error: %v", resp.StatusCode, err) } return nil, fmt.Errorf("expect response status code: %d, but got: %d. response body: %s", http.StatusOK, resp.StatusCode, body) } // encounter watch bytes received, does not expect to be broken if url == urlWatch || url == urlWatchWithGoaway { ch := make(chan watchResponse) go func() { defer resp.Body.Close() body := make([]byte, 0) buffer := make([]byte, 1) for { n, err := resp.Body.Read(buffer) if err != nil { // urlWatch will receive io.EOF, // urlWatchWithGoaway will receive http2.GoAwayError if err == io.EOF { err = nil } else if _, ok := err.(http2.GoAwayError); ok { err = nil } ch <- watchResponse{ body: body, err:  err, } return } body = append(body, buffer[0:n]...) } }() return ch, nil } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { return nil, fmt.Errorf("failed to read response body, error: %v", err) } if !reflect.DeepEqual(responseBody, body) { return nil, fmt.Errorf("expect response body: %s, got: %s", string(responseBody), string(body)) } return nil, nil } // TestClientReceivedGOAWAY tests the in-flight watch requests will not be affected and new requests use a new // connection after client received GOAWAY. func TestClientReceivedGOAWAY(t *testing.T) { s, err := newTestGOAWAYServer() if err != nil { t.Fatalf("failed to set-up test GOAWAY http server, err: %v", err) } s.StartTLS() defer s.Close() </ins> cases := []struct { name string reqs []string

Related discussion:
text: What happened: We enabled feature in our large-scale cluster, and we found client(Kubelet) may got error(500 http code response) like after received GOAWAY frame. I had already re-produce this bug and open an in golang repo. So far I think this bug only occur with a small probability when client request server in high concurrency use a same httpclient instance, and it's a golang sdk bug in http client. Before I solved this issue, I suggest do NOT enable GOAWAY() feature. As it's a golang sdk bug(currently I known and doubt) in http client, revert GOAWAY feature(codes) is not necessary.
cc
/sig api-machinery
Thanks for the report.
Based on not setting the field, I think it defaults that way, so that's good. We will leave the option and filter so that when golang updates its client, we will have a simple way to turn it on and execute it.
Sure. I am trying to fix the golang http client issue and will update progress at . After golang http client issue fixed, I will add more concurrent test cases to see if there are any other bugs may occur by GOAWAY feature. BTW, we still enable GOAWAY feature in our production cluster, because there is no any evidence told us this error will influence SLO, as all most components(kubelet, controller-manager, scheduler) is controller work mode and will re-try after got error. /assign
Golang version or required, which will release at 07/01/2020.
Kubernetes v1.19(or master branch) now is using golang 1.15, and golang 1.15 has fixed the issue. And is on going. Let me add more test cases to ensure the error will not happen again.
I think we should re-open this issue because of . /re-open