Change:
Using the Fabric8 client, which defaults to using WebSockets for Watches, I established a Watch on Service resources. After the timeout (implicit or explicit) rather than the WebSocket seeing a close control frame or even an EOF it continued to be open and fulfill ping/pong requests - but no Service events are sent. The WebSocket would close. Create a WebSocket based Watch with the ListOptions timeoutSeconds set on the Service resource along with a steady production of Service events, such as create/delete of a dummy Service. Kubernetes version (use ): Client Version: version.Info{Major:"1", Minor:"19", GitVersion:"v1.19.0", GitCommit:"", GitTreeState:"clean", BuildDate:"2020-10-27T05:15:27Z", GoVersion:"go1.15.0", Compiler:"gc", Platform:"linux/amd64"} Server Version: version.Info{Major:"1", Minor:"20", GitVersion:"v1.20.2", GitCommit:"", GitTreeState:"clean", BuildDate:"2021-01-13T13:20:00Z", GoVersion:"go1.15.5", Compiler:"gc", Platform:"linux/amd64"} The behavior has been seen in both local minikube and OpenShift distributions based upon Kubernetes 1.20. OS (e.g: ): Fedora 33
/sig api-machinery
Is this the same issue as ?
It could be related, but in that case the client is calling close. In this case we expect the server to initiate the close.
A couple of things were discussed when we triaged this issue: a) we need to involve the Fabric8 team (~java client?) b) We don't always default to Websocket, in fact we discussed discontinuing it c) But the code is still there, so it's allowed for now and we should support it. Not sure what to do next
a) no, it sounds like an apiserver problem, it should be reproducible with a go client in a test. b) If anything, we should standardize on websockets. Brower based clients depend on them. And when/if we get rid of SPDY, websockets are the replacement. This may be relevant to interests.
I didn't see an option to use websockets with the go client the impl is at Is there another code path or go client that will use a websocket for Watches?
Technically I didn't say our go client, just a go client. I just had in mind using the raw websockets library. must be testing the websocket watches somehow?
I adapted to It will simply keep running - with no events after the timeout. If you switch to configmap, it will terminate.
that's a terrific reproducer, thanks!
Yeah, we should absolutely be able to stick that in a unit / integration test somewhere, thanks!
opened in configmap watch works correctly, service watch hits timeout without the server closing the connection
I suspect the issue is in newDecoratedWatcher. Services implement a decorator and configmaps don't.
yup, the request context wasn't propagated to the decorated watcher
Service decorator was in 1.20 in Fix with unit and integration test in Will pick to 1.21 and 1.20 once merged
/triage accepted

Discussion:
resultCh  chan watch.Event } <del> func newDecoratedWatcher(w watch.Interface, decorator func(runtime.Object)) *decoratedWatcher { ctx, cancel := context.WithCancel(context.Background()) </del> <ins> func newDecoratedWatcher(ctx context.Context, w watch.Interface, decorator func(runtime.Object)) *decoratedWatcher { ctx, cancel := context.WithCancel(ctx) </ins> d := &decoratedWatcher{ w:         w, decorator: decorator,

return d } <ins> // run decorates watch events from the underlying watcher until its result channel // is closed or the passed in context is done. // When run() returns, decoratedWatcher#resultCh is closed. </ins> func (d *decoratedWatcher) run(ctx context.Context) { var recv, send watch.Event var ok bool <ins> defer close(d.resultCh) </ins> for { select { case recv, ok = <-d.w.ResultChan(): <del> // The underlying channel may be closed after timeout. </del> if !ok { <ins> // The underlying channel was closed, cancel our context </ins> d.cancel() return }

} select { case d.resultCh <- send: <del> if send.Type == watch.Error { d.cancel() } </del> <ins> // propagated event successfully </ins> case <-ctx.Done(): <ins> // context timed out or was cancelled, stop the underlying watcher d.w.Stop() return </ins> } case <-ctx.Done(): <ins> // context timed out or was cancelled, stop the underlying watcher </ins> d.w.Stop() <del> close(d.resultCh) </del> return } } } func (d *decoratedWatcher) Stop() { <ins> // stop the underlying watcher d.w.Stop() // cancel our context </ins> d.cancel() }

package registry import ( <ins> "context" </ins> "testing" "time"

func TestDecoratedWatcher(t *testing.T) { w := watch.NewFake() decorator := func(obj runtime.Object) { <del> pod := obj.(*example.Pod) pod.Annotations = map[string]string{"decorated": "true"} </del> <ins> if pod, ok := obj.(*example.Pod); ok { pod.Annotations = map[string]string{"decorated": "true"} } </ins> } <del> dw := newDecoratedWatcher(w, decorator) </del> <ins> ctx, cancel := context.WithCancel(context.Background()) dw := newDecoratedWatcher(ctx, w, decorator) </ins> defer dw.Stop() <del> go w.Add(&example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) </del> <ins> go func() { w.Error(&metav1.Status{Status: "Failure"}) w.Add(&example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) w.Error(&metav1.Status{Status: "Failure"}) w.Modify(&example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) w.Error(&metav1.Status{Status: "Failure"}) w.Delete(&example.Pod{ObjectMeta: metav1.ObjectMeta{Name: "foo"}}) }() expectErrorEvent(t, dw) // expect error is plumbed and doesn't force close the watcher expectPodEvent(t, dw, watch.Added) expectErrorEvent(t, dw) // expect error is plumbed and doesn't force close the watcher expectPodEvent(t, dw, watch.Modified) expectErrorEvent(t, dw) // expect error is plumbed and doesn't force close the watcher expectPodEvent(t, dw, watch.Deleted) // cancel the passed-in context to simulate request timeout cancel() // expect the decorated channel to be closed select { case e, ok := <-dw.ResultChan(): if ok { t.Errorf("expected result chan closed, got %#v", e) } case <-time.After(wait.ForeverTestTimeout): t.Errorf("timeout after %v", wait.ForeverTestTimeout) } // expect the underlying watcher to have been stopped as a result of the context cancellation if !w.IsStopped() { t.Errorf("expected underlying watcher to be stopped") } } func expectPodEvent(t *testing.T, dw *decoratedWatcher, watchType watch.EventType) { </ins> select { case e := <-dw.ResultChan(): pod, ok := e.Object.(*example.Pod) if !ok { <del> t.Errorf("Should received object of type *api.Pod, get type (%T)", e.Object) return </del> <ins> t.Fatalf("Should received object of type *api.Pod, get type (%T)", e.Object) </ins> } if pod.Annotations["decorated"] != "true" { <del> t.Errorf("pod.Annotations["decorated"], want=%s, get=%s", "true", pod.Labels["decorated"]) </del> <ins> t.Fatalf("pod.Annotations["decorated"], want=%s, get=%s", "true", pod.Labels["decorated"]) } if e.Type != watchType { t.Fatalf("expected type %s, got %s", watchType, e.Type) </ins> } case <-time.After(wait.ForeverTestTimeout): <del> t.Errorf("timeout after %v", wait.ForeverTestTimeout) </del> <ins> t.Fatalf("timeout after %v", wait.ForeverTestTimeout) } } func expectErrorEvent(t *testing.T, dw *decoratedWatcher) { select { case e := <-dw.ResultChan(): _, ok := e.Object.(*metav1.Status) if !ok { t.Fatalf("Should received object of type *metav1.Status, get type (%T)", e.Object) } if e.Type != watch.Error { t.Fatalf("expected type %s, got %s", watch.Error, e.Type) } case <-time.After(wait.ForeverTestTimeout): t.Fatalf("timeout after %v", wait.ForeverTestTimeout) </ins> } }

return nil, err } if e.Decorator != nil { <del> return newDecoratedWatcher(w, e.Decorator), nil </del> <ins> return newDecoratedWatcher(ctx, w, e.Decorator), nil </ins> } return w, nil }

package apimachinery import ( <ins> "bytes" </ins> "context" <ins> "io" "log" </ins> "net/http/httptest" "net/http/httputil" "net/url" <ins> "strings" </ins> "testing" "time" <ins> "golang.org/x/net/websocket" </ins> corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime"

"k8s.io/kubernetes/test/integration/framework" ) <ins> func TestWebsocketWatchClientTimeout(t *testing.T) { // server setup masterConfig := framework.NewIntegrationTestControlPlaneConfig() instance, s, closeFn := framework.RunAMaster(masterConfig) defer closeFn() // object setup service := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, Spec: corev1.ServiceSpec{ Ports: []corev1.ServicePort{{Name: "http", Port: 80}}, }, } configmap := &corev1.ConfigMap{ ObjectMeta: metav1.ObjectMeta{Name: "test"}, } clientset, err := kubernetes.NewForConfig(instance.GenericAPIServer.LoopbackClientConfig) if err != nil { t.Fatal(err) } if _, err := clientset.CoreV1().Services("default").Create(context.TODO(), service, metav1.CreateOptions{}); err != nil { t.Fatal(err) } if _, err := clientset.CoreV1().ConfigMaps("default").Create(context.TODO(), configmap, metav1.CreateOptions{}); err != nil { t.Fatal(err) } testcases := []struct { name         string path         string timeout      time.Duration expectResult string }{ { name:         "configmaps", path:         "/api/v1/configmaps?watch=true&timeoutSeconds=5", timeout:      10 * time.Second, expectResult: `"name":"test"`, }, { name:         "services", path:         "/api/v1/services?watch=true&timeoutSeconds=5", timeout:      10 * time.Second, expectResult: `"name":"test"`, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { u, _ := url.Parse(s.URL) apiURL := "ws://" + u.Host + tc.path wsc, err := websocket.NewConfig(apiURL, apiURL) if err != nil { log.Fatal(err) } wsConn, err := websocket.DialConfig(wsc) if err != nil { t.Fatal(err) } defer wsConn.Close() resultCh := make(chan string) go func() { defer close(resultCh) buf := &bytes.Buffer{} for { var msg []byte if err := websocket.Message.Receive(wsConn, &msg); err != nil { if err == io.EOF { resultCh <- buf.String() return } if !t.Failed() { // if we didn't already fail, treat this as an error t.Errorf("Failed to read completely from websocket %v", err) } return } if len(msg) == 0 { t.Logf("zero-length message") continue } t.Logf("Read %v %v", len(msg), string(msg)) buf.Write(msg) } }() select { case resultString := <-resultCh: if !strings.Contains(resultString, tc.expectResult) { t.Fatalf("Unexpected result:n%s", resultString) } case <-time.After(tc.timeout): t.Fatalf("hit timeout before connection closed") } }) } } </ins> func TestWatchClientTimeout(t *testing.T) { masterConfig := framework.NewIntegrationTestControlPlaneConfig() _, s, closeFn := framework.RunAMaster(masterConfig)