Change:
The test case "executing a command with run and attach without stdin" is inherently racy. The test runs, and expected to see in the output. The case when the test passes: case 1: Kubernetes creates and starts the container without stdin; The container exits before attach runs; kubectl falls back to get container log, which contains . case 2: Kubernetes creates and starts the container without stdin; kubectl attach before "stdin closed" is printed; kubectl attach contains . The case when the test fails: Kubernetes creates and starts the container without stdin; The container prints "stdin closed", but hasn't exited yet; kubectl attach contains nothing. One example failure: Related issue: Can we modify this test not to check the output? Or use to check output instead? /cc
Uploaded to try to address
/reopen This test is still inherently racy, even after
Fix in (I hope!)
Moving into the current milestone as it was not fixed in 1.14
Hello I'm a bug triage shadow for the 1.16 release cycle and considering this issue is tagged for 1.16, I'd like to check its status. Code freeze is starting on August 29th (about 3 days from now). Do we still target this issue to be fixed for 1.16?
Thanks . PR is available and awaiting review!
What is the PR in question? Is it targeting 1.16 or 1.17?
/reopen This test is having a lot of failures these days
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
is right, it seems to be present in kind jobs now only
/assign let's see if we start to recover a bit of stability :smile:
The rate of failures decreased a lot since last changes Checking the logs of the failures it seems that the problem is that the pod has not finished before checking the output, as Lantao described The test fails Collecting events from namespace The framework logs the pods the kubelet thinks is on the node
note that it's still racy, but it's hopefully somewhat mitigated.
/reopen
Reopened this issue. <detailsIn response to : /reopen Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
any updates here?
nope, totally agree with the description, is racy by design, couldn't find any way or trick to avoid the randomness
EDIT: It's late and I'm actually not sure I'm remembering this correctly, take this comment with a heap of salt: Specific sub-tests were fixed previously I think, but this one sub-case: That one we can't use the "Wait for stdin input to signal we're attached" hack, so it's just racy and broken. We could remove that case maybe. I'm not sure who I'd want to weigh in on that, probably someone more active in node / CLI.
not super common atm but still happening. discussing
/unassign has a fix for this

Discussion:
}) ginkgo.It("should support inline execution and attach", func() { <ins> waitForStdinContent := func(pod, content string) string { var logOutput string err := wait.Poll(10*time.Second, 5*time.Minute, func() (bool, error) { logOutput = framework.RunKubectlOrDie(ns, "logs", pod) return strings.Contains(logOutput, content), nil }) gomega.Expect(err).To(gomega.BeNil(), fmt.Sprintf("unexpected error waiting for '%v' output", content)) return logOutput } </ins> ginkgo.By("executing a command with run and attach with stdin") // We wait for a non-empty line so we know kubectl has attached <del> runOutput := framework.NewKubectlCommand(ns, "run", "run-test", "--image="+busyboxImage, "--restart=OnFailure", "--attach=true", "--stdin", "--", "sh", "-c", "while [ -z "$s" ]; do read s; sleep 1; done; echo read:$s && cat && echo 'stdin closed'"). </del> <ins> framework.NewKubectlCommand(ns, "run", "run-test", "--image="+busyboxImage, "--restart=OnFailure", "--attach=true", "--stdin", "--", "sh", "-c", "echo -n read: && cat && echo 'stdin closed'"). </ins> WithStdinData("valuenabcd1234"). ExecOrDie(ns) <ins> runOutput := waitForStdinContent("run-test", "stdin closed") </ins> gomega.Expect(runOutput).To(gomega.ContainSubstring("read:value")) gomega.Expect(runOutput).To(gomega.ContainSubstring("abcd1234")) gomega.Expect(runOutput).To(gomega.ContainSubstring("stdin closed"))

// "stdin closed", but hasn't exited yet. // We wait 10 seconds before printing to give time to kubectl to attach // to the container, this does not solve the race though. <del> runOutput = framework.NewKubectlCommand(ns, "run", "run-test-2", "--image="+busyboxImage, "--restart=OnFailure", "--attach=true", "--leave-stdin-open=true", "--", "sh", "-c", "sleep 10; cat && echo 'stdin closed'"). </del> <ins> framework.NewKubectlCommand(ns, "run", "run-test-2", "--image="+busyboxImage, "--restart=OnFailure", "--attach=true", "--leave-stdin-open=true", "--", "sh", "-c", "cat && echo 'stdin closed'"). </ins> WithStdinData("abcd1234"). ExecOrDie(ns) <ins> runOutput = waitForStdinContent("run-test-2", "stdin closed") </ins> gomega.Expect(runOutput).ToNot(gomega.ContainSubstring("abcd1234")) gomega.Expect(runOutput).To(gomega.ContainSubstring("stdin closed")) gomega.Expect(c.CoreV1().Pods(ns).Delete(context.TODO(), "run-test-2", metav1.DeleteOptions{})).To(gomega.BeNil()) ginkgo.By("executing a command with run and attach with stdin with open stdin should remain running") <del> runOutput = framework.NewKubectlCommand(ns, "run", "run-test-3", "--image="+busyboxImage, "--restart=OnFailure", "--attach=true", "--leave-stdin-open=true", "--stdin", "--", "sh", "-c", "cat && echo 'stdin closed'"). </del> <ins> framework.NewKubectlCommand(ns, "run", "run-test-3", "--image="+busyboxImage, "--restart=OnFailure", "--attach=true", "--leave-stdin-open=true", "--stdin", "--", "sh", "-c", "cat && echo 'stdin closed'"). </ins> WithStdinData("abcd1234n"). ExecOrDie(ns) <ins> runOutput = waitForStdinContent("run-test-3", "abcd1234") gomega.Expect(runOutput).To(gomega.ContainSubstring("abcd1234")) </ins> gomega.Expect(runOutput).ToNot(gomega.ContainSubstring("stdin closed")) g := func(pods []*v1.Pod) sort.Interface { return sort.Reverse(controller.ActivePods(pods)) } runTestPod, _, err := polymorphichelpers.GetFirstPod(f.ClientSet.CoreV1(), ns, "run=run-test-3", 1*time.Minute, g) framework.ExpectNoError(err)

framework.Failf("Pod %q of Job %q should still be running", runTestPod.Name, "run-test-3") } <del> // NOTE: we cannot guarantee our output showed up in the container logs before stdin was closed, so we have // to loop test. err = wait.PollImmediate(time.Second, time.Minute, func() (bool, error) { if !e2epod.CheckPodsRunningReady(c, ns, []string{runTestPod.Name}, 1*time.Second) { framework.Failf("Pod %q of Job %q should still be running", runTestPod.Name, "run-test-3") } logOutput := framework.RunKubectlOrDie(ns, "logs", runTestPod.Name) gomega.Expect(logOutput).ToNot(gomega.ContainSubstring("stdin closed")) return strings.Contains(logOutput, "abcd1234"), nil }) framework.ExpectNoError(err) </del> gomega.Expect(c.CoreV1().Pods(ns).Delete(context.TODO(), "run-test-3", metav1.DeleteOptions{})).To(gomega.BeNil()) })