Change:
<ins> /* Copyright 2024 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package apimachinery import ( "context" "fmt" "testing" "time" corev1 "k8s.io/api/core/v1" apiequality "k8s.io/apimachinery/pkg/api/equality" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/kubernetes" kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing" "k8s.io/kubernetes/test/integration/framework" ) func TestWatchNamespaceEvent(t *testing.T) { timeout := 30 * time.Second server := kubeapiservertesting.StartTestServerOrDie(t, nil, nil, framework.SharedEtcd()) defer server.TearDownFn() clientSet, err := kubernetes.NewForConfig(server.ClientConfig) if err != nil { t.Fatal(err) } newWatcher := func(ctx context.Context, ns *corev1.Namespace, resourceVersion string) (watch.Interface, error) { return clientSet.CoreV1().Namespaces().Watch(ctx, metav1.ListOptions{ FieldSelector:   fields.OneTermEqualSelector("metadata.name", ns.Name).String(), ResourceVersion: resourceVersion, }) } newLegacyNSWatcher := func(ctx context.Context, ns *corev1.Namespace, resourceVersion string) (watch.Interface, error) { if resourceVersion == "" { return clientSet.CoreV1().RESTClient().Get(). RequestURI(fmt.Sprintf("/api/v1/watch/namespaces/%s", ns.Name)).Watch(ctx) } else { return clientSet.CoreV1().RESTClient().Get(). RequestURI(fmt.Sprintf("/api/v1/watch/namespaces/%s?resourceVersion=%s", ns.Name, resourceVersion)).Watch(ctx) } } newTestNamespace := func(name string) *corev1.Namespace { return &corev1.Namespace{ ObjectMeta: metav1.ObjectMeta{ Name: name, }, } } tt := []struct { name            string namespace       *corev1.Namespace resourceVersion string getWatcher      func(ctx context.Context, ns *corev1.Namespace, resourceVersion string) (watch.Interface, error) }{ { name:       "watch namespace", namespace:  newTestNamespace("watch-namespace"), getWatcher: newWatcher, }, { name:            "watch namespace with resource version", namespace:       newTestNamespace("watch-ns-with-rv"), getWatcher:      newWatcher, resourceVersion: "0", }, { name:       "legacy watch namespace api", namespace:  newTestNamespace("legacy-watch-ns"), getWatcher: newLegacyNSWatcher, }, { name:            "legacy watch namespace api with resource version", namespace:       newTestNamespace("legacy-watch-ns-with-rv"), getWatcher:      newLegacyNSWatcher, resourceVersion: "0", }, } t.Run("group", func(t *testing.T) { for _, tc := range tt { tc := tc // we need to copy it for parallel runs t.Run(tc.name, func(t *testing.T) { t.Parallel() startTest := time.Now() ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() ns, err := clientSet.CoreV1().Namespaces().Create(ctx, tc.namespace, metav1.CreateOptions{}) if err != nil { t.Fatalf("Failed to create testing namespace %s: %v", tc.namespace.Name, err) } else { defer framework.DeleteNamespaceOrDie(clientSet, ns, t) } watcher, err := tc.getWatcher(ctx, ns, tc.resourceVersion) if err != nil { t.Fatalf("Failed to create watcher: %v", err) } defer watcher.Stop() t.Logf("start watching namespace %s event", tc.namespace.Name) generateAndWatchNamespaceEvent(ctx, t, clientSet, ns, watcher) t.Logf("Watch duration: %v", time.Since(startTest)) }) } }) } func generateAndWatchNamespaceEvent(ctx context.Context, t *testing.T, clientSet *kubernetes.Clientset, ns *corev1.Namespace, watcher watch.Interface) { timeout := 10 * time.Second nsName := ns.Name t.Logf("Expectiong to watch an add event") _, ok := waitForEvent(watcher, watch.Added, ns, timeout) if !ok { t.Fatalf("Failed to watch add event") } if ns.Annotations == nil { ns.Annotations = map[string]string{"e2e-watch-namespace": "update-v1"} } else { ns.Annotations["e2e-watch-namespace"] = "update-v1" } ns, err := clientSet.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}) if err != nil { t.Fatalf("Failed to update namespace %s: %v", nsName, err) } t.Logf("Expectiong to watch an update event") _, ok = waitForEvent(watcher, watch.Modified, ns, timeout) if !ok { t.Fatalf("Failed to watch update event") } ns.Annotations["e2e-watch-namespace"] = "update-v2" ns, err = clientSet.CoreV1().Namespaces().Update(ctx, ns, metav1.UpdateOptions{}) if err != nil { t.Fatalf("failed to delete namespace %s: %v", nsName, err) } t.Logf("Expectiong to watch an update event") _, ok = waitForEvent(watcher, watch.Modified, ns, timeout) if !ok { t.Fatalf("Failed to watch update event") } } func waitForEvent(w watch.Interface, expectType watch.EventType, expectObject runtime.Object, duration time.Duration) (watch.Event, bool) { stopTimer := time.NewTimer(duration) defer stopTimer.Stop() for { select { case actual, ok := <-w.ResultChan(): if !ok { return watch.Event{}, false } if expectType == actual.Type && (expectObject == nil || apiequality.Semantic.DeepEqual(expectObject, actual.Object)) { return actual, true } case <-stopTimer.C: expected := watch.Event{ Type:   expectType, Object: expectObject, } return expected, false } } } </ins>

Discussion:
We upgraded our test cluster to (GKE ) and noticed that some of our tests began to time out when watching for events that never arrived. Upon investigation, it appears that there is a regression in the deprecated path-based watch API for at least e.g. . We have not been able to reproduce this with any other resource type. When creating a watch for a namespace (see ) we see the initial event, but then we do not see subsequent events if we modify the namespace e.g. add a label or annotation. I would expect to see events when the namespace is altered. E.g. adding a label should produce a event with the contents. a new test namespace e.g. a proxy to the master the watch path of the namespace that has been created e.g. another shell, modify the namespace to add a label to it. that there is no modified event in the watch. It is understood that the based paths are and have been for some time deprecated, we are working on moving to use the watch query param instead. <detailsWe are using Google Kubernetes Engine, please see version above. </detailsNo response No response No response No response
/sig api-machinery
The addWatcher add watch to allWatchers with with the value namespace not empty. But the function dispatch event with an empty namespace.
This issue is limited to watchers of the core namespace resource using the deprecated endpoint and watching a single namespace. This was introduced in 1.27 in (which was merged with test coverage in 1.27 in )
Hi, just to confirm this was working fine (and still is on our production clusters) on 1.28.7 it's only since 1.28.8 that we can see this bug, so slightly confused by your comment (and issue rename) to say 1.27+?
I reproduced the break at HEAD of release-1.27, and assumed the break came in with , but I was wrong.

release-1.26 HEAD: works v1.27.0: works release-1.27 HEAD: broken v1.28.0: works v1.28.7: works per your feedback v1.28.8: breaks per your feedback release-1.28 HEAD: broken That means that watch cache changes cherry picked back to 1.27 and 1.28 regressed something, and I don't know exactly what the breaking change was. The only thing that looks at all relevant between 1.28.7 and 1.28.8 is , but I don't see how that would only impact the single-item watch cc
Did you send the request with a resource version, or any other parameter? If the parameter is not empty, will work, or the old watcher will work. This makes a big difference.
This commit add the feature gate .
No, I just sent and that broke somewhere between 1.27.0 and
If you are asking me, I did not send any additional query parameter, no. On our cluster: works as expected, we see events when we add labels etc. On our cluster: does not work, we do not see events.
Ok, figured it out... 27.0 which is what made the watch cache not handle correctly (merged in with test coverage) also which routed watch requests without a resourceVersion parameter directly to etcd, bypassing watch cache 27.13 / 1.28.9 / 1.29.4 / 1.30.0 moved the behavior from behind an off-by-default gate because it was a massive performance regression. This had the side effect of making watch requests without a resourceVersion parameter get handled by the watch cache, and get exposed to this issue.

(GKE accelerated that specific fix internally to resolve that performance regression, so it showed up in a GKE 1.28.8-gke.X build, which is what confused me when looking for the delta between v1.28.7 and v1.28.8) in [1.27.0-1.12.12], [1.28.0-1.28.8], [1.29-1.29.3]: routes to etcd and works (because of ) routes to etcd and works (because of ) routes to watch cache and fails (where X is not 0) routes to watch cache and failsin 1.27.13+, 1.28.9+, 1.29.4+, 1.30.0+: routes to watch cache and fails routes to watch cache and fails routes to watch cache and fails (where X is not 0) routes to watch cache and fails
The workaround for anyone encountering this issue is to use the non-deprecated watch endpoint (and is a generally good change to make anyway) The Kubernetes fix for this I would recommend to restore pre-1.27 behavior is in my comment in Once that is done and test coverage , I'd be happy to see that backported to 1.27+
/cc
GitHub didn't allow me to assign the following users: tinatingyu. Note that only with read permissions, repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign /cc Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/assign
GitHub didn't allow me to assign the following users: yt2985. Note that only with read permissions, repo collaborators and people who have commented on this issue/PR can be assigned. Additionally, issues/PRs can only have 10 assignees at the same time. For more information please see <detailsIn response to : /assign Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
just to verify, are you planning to take to completion?

if so, it would be ideal to complete that so the 1.27 backport can happen before 1.27 reaches EOL in June
I saw I was assigned with this issue. Taking a look...
sorry, was tagging you in since you helped bring in and might have context, but realized I should check with first since they have a fix in progress
Yew, working on it and writing test case.
cc
is this issue resolved already?
Not yet, is the proposed fix, under review now