Change:
This Issue replace Watch events may be missed and the test may still pass. We need to ensure that all expected watch events are seen. Watch events are not collected and checked at the end of the test. In order to solve the problem, we must collect all watch events as they come in and perform a check at the end of the test to ensure that the order is correct and they contain no errors.
/sig testing /area conformance /assign /cc
This Issue replace
AI: Must allow for other events to occur through the expected watch events stream expected watch events: must allow the following to be valid:
[x] Set up a watch for resource through a static label to the resource with DynamicClient [x] Manage the watch if it dies [x] Feed the watch to the function [x] Read all the watch events from the at the end of the test, ensuring that the events occur question: What does doing this mean to tests which check data? (e.g: - waits for values, sourced from watch) [x] Retry the test, upon failure [x] Manage the resources on retry (delete)
More or less, yes. If we need to read results mid-test, I think that will be OK, rather than waiting for the end. The main point is to wrap the watch such that it gets restarted if it fails. That wrapper will need to sit between the raw watch channel and the test: raw watch -channel -wrapper (restarts watch if closes/fails prematurely) -channel consumed in test I'm a little surprised there isn't already a sort of "buffered watch" object that does this already ? You'll also need a stop channel for the go routine started by the wrapper, so that at the end of the test you can stop that go routine and watch.
Also, if we have the "protected watch" wrapper, not sure we need to retry the test on failure. AFAIU, the only reason we wouldn't see the watch events we expected would be because of etcd compaction, which would not be expected to happen in e2e clusters, and even if it did it would not happen to event histories that are brand new (i.e., it only compacts old history). Again would like to confirm.
/reopen While we did push through this first iteration, this ticket is for ongoing discussion of watch tooling and refactoring of the approach.
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen While we did push through this first iteration, this ticket is for ongoing discussion of watch tooling and refactoring of the approach.
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
"//staging/src/k8s.io/apimachinery/pkg/fields:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/labels:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/runtime:go_default_library", <ins> "//staging/src/k8s.io/apimachinery/pkg/runtime/schema:go_default_library", </ins> "//staging/src/k8s.io/apimachinery/pkg/types:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/diff:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/intstr:go_default_library",

"//staging/src/k8s.io/apimachinery/pkg/util/uuid:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/util/wait:go_default_library", "//staging/src/k8s.io/apimachinery/pkg/watch:go_default_library", <ins> "//staging/src/k8s.io/client-go/dynamic:go_default_library", </ins> "//staging/src/k8s.io/client-go/kubernetes:go_default_library", "//staging/src/k8s.io/client-go/tools/cache:go_default_library", "//staging/src/k8s.io/client-go/tools/watch:go_default_library",

import ( "bytes" "context" <ins> "encoding/json" </ins> "fmt" "io" <ins> "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic" </ins> "runtime/debug" "strconv" "strings"

var _ = framework.KubeDescribe("Pods", func() { f := framework.NewDefaultFramework("pods") <ins> var dc dynamic.Interface </ins> var podClient *framework.PodClient ginkgo.BeforeEach(func() { podClient = f.PodClient() <ins> dc = f.DynamicClient </ins> }) /*

err = wait.PollImmediate(podRetryPeriod, podRetryTimeout, checkPodListQuantity(f, "type=Testing", 0)) framework.ExpectNoError(err, "found a pod(s)") }) <ins> ginkgo.It("should run through the lifecycle of Pods and PodStatus", func() { podResource := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} testNs := f.Namespace.Name testPodName := "pod-test" testPodImage := "nginx" testPodImage2 := "httpd" testPodLabels := map[string]string{"test-pod-static": "true"} testPodLabelsFlat := "test-pod-static=true" testPod := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:   testPodName, Labels: testPodLabels, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:  testPodName, Image: testPodImage, }, }, }, } ginkgo.By("creating a Pod with a static label") _, err := f.ClientSet.CoreV1().Pods(testNs).Create(context.TODO(), &testPod, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create Pod %s in namespace %s", testPodName, testNs) ginkgo.By("setting up a watch for the Pod") podWatchTimeoutSeconds := int64(180) podWatch, err := f.ClientSet.CoreV1().Pods(testNs).Watch(context.TODO(), metav1.ListOptions{LabelSelector: testPodLabelsFlat, TimeoutSeconds: &podWatchTimeoutSeconds}) framework.ExpectNoError(err, "failed to set up watch for Pod %s in namespace %s", testPodName, testNs) podWatchChan := podWatch.ResultChan() ginkgo.By("watching for Pod to be ready") eventFound := false for watchEvent := range podWatchChan { if podWatchEvent, ok := watchEvent.Object.(*v1.Pod); ok && podWatchEvent.Status.Phase == v1.PodRunning { eventFound = true break } } framework.ExpectEqual(eventFound, true, "failed to find running Pod %s in namespace %s", testPodName, testNs) ginkgo.By("patching the Pod with a new Label and updated data") podPatch, err := json.Marshal(v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"test-pod": "patched"}, }, Spec: v1.PodSpec{ Containers: []v1.Container{{ Name:  testPodName, Image: testPodImage2, }}, }, }) framework.ExpectNoError(err, "failed to marshal JSON patch for Pod") _, err = f.ClientSet.CoreV1().Pods(testNs).Patch(context.TODO(), testPodName, types.StrategicMergePatchType, []byte(podPatch), metav1.PatchOptions{}) framework.ExpectNoError(err, "failed to patch Pod %s in namespace %s", testPodName, testNs) ginkgo.By("getting the Pod and ensuring that it's patched") pod, err := f.ClientSet.CoreV1().Pods(testNs).Get(context.TODO(), testPodName, metav1.GetOptions{}) framework.ExpectNoError(err, "failed to fetch Pod %s in namespace %s", testPodName, testNs) framework.ExpectEqual(pod.ObjectMeta.Labels["test-pod"], "patched", "failed to patch Pod - missing label") framework.ExpectEqual(pod.Spec.Containers[0].Image, testPodImage2, "failed to patch Pod - wrong image") ginkgo.By("getting the PodStatus") podStatusUnstructured, err := dc.Resource(podResource).Namespace(testNs).Get(context.TODO(), testPodName, metav1.GetOptions{}, "status") framework.ExpectNoError(err, "failed to fetch PodStatus of Pod %s in namespace %s", testPodName, testNs) podStatusBytes, err := json.Marshal(podStatusUnstructured) framework.ExpectNoError(err, "failed to marshal unstructured response") var podStatus v1.Pod err = json.Unmarshal(podStatusBytes, &podStatus) framework.ExpectNoError(err, "failed to unmarshal JSON bytes to a Pod object type") ginkgo.By("replacing the Pod's status Ready condition to False") podStatusUpdated := podStatus podStatusFieldPatchCount := 0 podStatusFieldPatchCountTotal := 2 for pos, cond := range podStatusUpdated.Status.Conditions { if (cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue) || (cond.Type == v1.ContainersReady && cond.Status == v1.ConditionTrue) { podStatusUpdated.Status.Conditions[pos].Status = v1.ConditionFalse podStatusFieldPatchCount++ } } framework.ExpectEqual(podStatusFieldPatchCount, podStatusFieldPatchCountTotal, "failed to patch all relevant Pod conditions") podStatusUpdate, err := f.ClientSet.CoreV1().Pods(testNs).UpdateStatus(context.TODO(), &podStatusUpdated, metav1.UpdateOptions{}) framework.ExpectNoError(err, "failed to update PodStatus of Pod %s in namespace %s", testPodName, testNs) ginkgo.By("check the Pod again to ensure its Ready conditions are False") podStatusFieldPatchCount = 0 podStatusFieldPatchCountTotal = 2 for _, cond := range podStatusUpdate.Status.Conditions { if (cond.Type == v1.PodReady && cond.Status == v1.ConditionFalse) || (cond.Type == v1.ContainersReady && cond.Status == v1.ConditionFalse) { podStatusFieldPatchCount++ } } framework.ExpectEqual(podStatusFieldPatchCount, podStatusFieldPatchCountTotal, "failed to update PodStatus - field patch count doesn't match the total") ginkgo.By("deleting the Pod via a Collection with a LabelSelector") err = f.ClientSet.CoreV1().Pods(testNs).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: testPodLabelsFlat}) framework.ExpectNoError(err, "failed to delete Pod by collection") ginkgo.By("watching for the Pod to be deleted") eventFound = false for watchEvent := range podWatchChan { if watchEvent.Type == watch.Deleted { eventFound = true break } } framework.ExpectEqual(eventFound, true, "failed to find Pod %v event", watch.Deleted) }) </ins> }) func checkPodListQuantity(f *framework.Framework, label string, quantity int) func() (bool, error) {