Change:
err = wait.PollImmediate(podRetryPeriod, podRetryTimeout, checkPodListQuantity(f, "type=Testing", 0)) framework.ExpectNoError(err, "found a pod(s)") }) <ins> ginkgo.It("should run through the lifecycle of Pods and PodStatus", func() { podResource := schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} testNs := f.Namespace.Name testPodName := "pod-test" testPodImage := "nginx" testPodImage2 := "httpd" testPodLabels := map[string]string{"test-pod-static": "true"} testPodLabelsFlat := "test-pod-static=true" testPod := v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Name:   testPodName, Labels: testPodLabels, }, Spec: v1.PodSpec{ Containers: []v1.Container{ { Name:  testPodName, Image: testPodImage, }, }, }, } ginkgo.By("creating a Pod with a static label") _, err := f.ClientSet.CoreV1().Pods(testNs).Create(context.TODO(), &testPod, metav1.CreateOptions{}) framework.ExpectNoError(err, "failed to create Pod %s in namespace %s", testPodName, testNs) ginkgo.By("setting up a watch for the Pod") podWatchTimeoutSeconds := int64(180) podWatch, err := f.ClientSet.CoreV1().Pods(testNs).Watch(context.TODO(), metav1.ListOptions{LabelSelector: testPodLabelsFlat, TimeoutSeconds: &podWatchTimeoutSeconds}) framework.ExpectNoError(err, "failed to set up watch for Pod %s in namespace %s", testPodName, testNs) podWatchChan := podWatch.ResultChan() ginkgo.By("watching for Pod to be ready") eventFound := false for watchEvent := range podWatchChan { if podWatchEvent, ok := watchEvent.Object.(*v1.Pod); ok && podWatchEvent.Status.Phase == v1.PodRunning { eventFound = true break } } framework.ExpectEqual(eventFound, true, "failed to find running Pod %s in namespace %s", testPodName, testNs) ginkgo.By("patching the Pod with a new Label and updated data") podPatch, err := json.Marshal(v1.Pod{ ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{"test-pod": "patched"}, }, Spec: v1.PodSpec{ Containers: []v1.Container{{ Name:  testPodName, Image: testPodImage2, }}, }, }) framework.ExpectNoError(err, "failed to marshal JSON patch for Pod") _, err = f.ClientSet.CoreV1().Pods(testNs).Patch(context.TODO(), testPodName, types.StrategicMergePatchType, []byte(podPatch), metav1.PatchOptions{}) framework.ExpectNoError(err, "failed to patch Pod %s in namespace %s", testPodName, testNs) ginkgo.By("getting the Pod and ensuring that it's patched") pod, err := f.ClientSet.CoreV1().Pods(testNs).Get(context.TODO(), testPodName, metav1.GetOptions{}) framework.ExpectNoError(err, "failed to fetch Pod %s in namespace %s", testPodName, testNs) framework.ExpectEqual(pod.ObjectMeta.Labels["test-pod"], "patched", "failed to patch Pod - missing label") framework.ExpectEqual(pod.Spec.Containers[0].Image, testPodImage2, "failed to patch Pod - wrong image") ginkgo.By("getting the PodStatus") podStatusUnstructured, err := dc.Resource(podResource).Namespace(testNs).Get(context.TODO(), testPodName, metav1.GetOptions{}, "status") framework.ExpectNoError(err, "failed to fetch PodStatus of Pod %s in namespace %s", testPodName, testNs) podStatusBytes, err := json.Marshal(podStatusUnstructured) framework.ExpectNoError(err, "failed to marshal unstructured response") var podStatus v1.Pod err = json.Unmarshal(podStatusBytes, &podStatus) framework.ExpectNoError(err, "failed to unmarshal JSON bytes to a Pod object type") ginkgo.By("replacing the Pod's status Ready condition to False") podStatusUpdated := podStatus podStatusFieldPatchCount := 0 podStatusFieldPatchCountTotal := 2 for pos, cond := range podStatusUpdated.Status.Conditions { if (cond.Type == v1.PodReady && cond.Status == v1.ConditionTrue) || (cond.Type == v1.ContainersReady && cond.Status == v1.ConditionTrue) { podStatusUpdated.Status.Conditions[pos].Status = v1.ConditionFalse podStatusFieldPatchCount++ } } framework.ExpectEqual(podStatusFieldPatchCount, podStatusFieldPatchCountTotal, "failed to patch all relevant Pod conditions") podStatusUpdate, err := f.ClientSet.CoreV1().Pods(testNs).UpdateStatus(context.TODO(), &podStatusUpdated, metav1.UpdateOptions{}) framework.ExpectNoError(err, "failed to update PodStatus of Pod %s in namespace %s", testPodName, testNs) ginkgo.By("check the Pod again to ensure its Ready conditions are False") podStatusFieldPatchCount = 0 podStatusFieldPatchCountTotal = 2 for _, cond := range podStatusUpdate.Status.Conditions { if (cond.Type == v1.PodReady && cond.Status == v1.ConditionFalse) || (cond.Type == v1.ContainersReady && cond.Status == v1.ConditionFalse) { podStatusFieldPatchCount++ } } framework.ExpectEqual(podStatusFieldPatchCount, podStatusFieldPatchCountTotal, "failed to update PodStatus - field patch count doesn't match the total") ginkgo.By("deleting the Pod via a Collection with a LabelSelector") err = f.ClientSet.CoreV1().Pods(testNs).DeleteCollection(context.TODO(), metav1.DeleteOptions{}, metav1.ListOptions{LabelSelector: testPodLabelsFlat}) framework.ExpectNoError(err, "failed to delete Pod by collection") ginkgo.By("watching for the Pod to be deleted") eventFound = false for watchEvent := range podWatchChan { if watchEvent.Type == watch.Deleted { eventFound = true break } } framework.ExpectEqual(eventFound, true, "failed to find Pod %v event", watch.Deleted) }) </ins> }) func checkPodListQuantity(f *framework.Framework, label string, quantity int) func() (bool, error) {

Discussion:
This Issue replace Watch events may be missed and the test may still pass. We need to ensure that all expected watch events are seen. Watch events are not collected and checked at the end of the test. In order to solve the problem, we must collect all watch events as they come in and perform a check at the end of the test to ensure that the order is correct and they contain no errors.
/sig testing /area conformance /assign /cc
This Issue replace
AI: Must allow for other events to occur through the expected watch events stream expected watch events: must allow the following to be valid:
[x] Set up a watch for resource through a static label to the resource with DynamicClient [x] Manage the watch if it dies [x] Feed the watch to the function [x] Read all the watch events from the at the end of the test, ensuring that the events occur question: What does doing this mean to tests which check data? (e.g: - waits for values, sourced from watch) [x] Retry the test, upon failure [x] Manage the resources on retry (delete)
More or less, yes. If we need to read results mid-test, I think that will be OK, rather than waiting for the end. The main point is to wrap the watch such that it gets restarted if it fails. That wrapper will need to sit between the raw watch channel and the test: raw watch -channel -wrapper (restarts watch if closes/fails prematurely) -channel consumed in test I'm a little surprised there isn't already a sort of "buffered watch" object that does this already ? You'll also need a stop channel for the go routine started by the wrapper, so that at the end of the test you can stop that go routine and watch.
Also, if we have the "protected watch" wrapper, not sure we need to retry the test on failure. AFAIU, the only reason we wouldn't see the watch events we expected would be because of etcd compaction, which would not be expected to happen in e2e clusters, and even if it did it would not happen to event histories that are brand new (i.e., it only compacts old history). Again would like to confirm.
/reopen While we did push through this first iteration, this ticket is for ongoing discussion of watch tooling and refactoring of the approach.
Reopened this issue.

<detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/reopen While we did push through this first iteration, this ticket is for ongoing discussion of watch tooling and refactoring of the approach.
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

[x] Original approval issue: - Reverted due to flakes [x] Two weeks soak end date: 20 November 2020 This issue is created to allow edit by /wip /hold According to this APIsnoop query, there are still some remaining Pod endpoints which are untested. a Pod with a static label the Pod with a new Label and updated data the Pod to ensure it's patched the Pod's status the Pod's status Ready condition to False the Pod's status to check if Ready is False all Pods in all Namespaces find the Pod(1) ensure that the Pod is found and is patched Namespaced Pod(1) via a Collection with a LabelSelector the Pod to check that it's deleted Discover useragents: List endpoints hit by the test: Display endpoint coverage change: If a test with these calls gets merged, test coverage will go up by 4 points This test is also created with the goal of conformance promotion. /sig testing /sig architecture /area conformance
/reopen Test PR Merge Promotion PR to be created next
Reopened this issue. <detailsIn response to : /reopen Test PR Merge Promotion PR to be created next Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
reverted by due to flaky test
Issues go stale after 90d of inactivity. Mark the issue as fresh with . Stale issues rot after an additional 30d of inactivity and eventually close. If this issue is safe to close now please do so with . Send feedback to sig-testing, kubernetes/test-infra and/or . /lifecycle stale
/remove-lifecycle stale
/reopen PR on test grid for 2 weeks before promotion to conformance
Reopened this issue. <detailsIn response to : /reopen PR on test grid for 2 weeks before promotion to conformance Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/triage accepted