Code change:
Slow conformance test has caused issues like We should make runtime/image conformance test run faster without breaking the test coverage, possibly by: Shorten consistently check interval . Remove redundant test cases such as ... /cc

Related discussion:
text: }, }, } <del> testCount := int32(3) testStatuses := []testStatus{ {"terminate-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateWaiting | ContainerStateRunning | ContainerStateTerminated, ">", testCount, false}, {"terminate-cmd-rpof", api.RestartPolicyOnFailure, api.PodSucceeded, ContainerStateTerminated, "==", testCount, false}, {"terminate-cmd-rpn", api.RestartPolicyNever, api.PodSucceeded, ContainerStateTerminated, "==", 0, false}, </del> <ins> testCases := []testCase{ {"terminate-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateRunning, "==", 2, true}, {"terminate-cmd-rpof", api.RestartPolicyOnFailure, api.PodSucceeded, ContainerStateTerminated, "==", 1, false}, {"terminate-cmd-rpn", api.RestartPolicyNever, api.PodFailed, ContainerStateTerminated, "==", 0, false}, </ins> } <del> for _, testStatus := range testStatuses { </del> <ins> for _, testCase := range testCases { </ins> tmpFile, err := ioutil.TempFile("", "restartCount") Expect(err).NotTo(HaveOccurred()) defer os.Remove(tmpFile.Name()) <del> // It fails in the first three runs and succeeds after that. tmpCmd := fmt.Sprintf("echo 'hello' >> /restart-count/%s ; test $(wc -l /restart-count/%s| awk {'print $1'}) -ge %d", path.Base(tmpFile.Name()), path.Base(tmpFile.Name()), testCount+1) testContainer.Name = testStatus.Name </del> <ins> // It failed at the 1st run, then succeeded at 2nd run, then run forever cmdScripts := ` f=%s count=$(echo 'hello' >> $f ; wc -l $f | awk {'print $1'}) if [ $count -eq 1 ]; then exit 1 fi if [ $count -eq 2 ]; then exit 0 fi while true; do sleep 1; done ` tmpCmd := fmt.Sprintf(cmdScripts, path.Join(restartCountVolumePath, path.Base(tmpFile.Name()))) testContainer.Name = testCase.Name </ins> testContainer.Command = []string{"sh", "-c", tmpCmd} terminateContainer := ConformanceContainer{ Container:     testContainer, Client:        cl, <del> RestartPolicy: testStatus.RestartPolicy, </del> <ins> RestartPolicy: testCase.RestartPolicy, </ins> Volumes:       testVolumes, NodeName:      *nodeName, Namespace:     namespace, } <del> err = terminateContainer.Create() Expect(err).NotTo(HaveOccurred()) testCContainers = append(testCContainers, terminateContainer) </del> <ins> Expect(terminateContainer.Create()).To(Succeed()) defer terminateContainer.Delete() </ins> <del> Eventually(func() api.PodPhase { _, phase, _ := terminateContainer.GetStatus() return phase }, retryTimeout, pollInterval).ShouldNot(Equal(api.PodPending)) var status api.ContainerStatus </del> By("it should get the expected 'RestartCount'") <del> Eventually(func() int32 { status, _, _ = terminateContainer.GetStatus() return status.RestartCount }, retryTimeout, pollInterval).Should(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) </del> <ins> Eventually(func() (int32, error) { status, err := terminateContainer.GetStatus() return status.RestartCount, err }, retryTimeout, pollInterval).Should(BeNumerically(testCase.RestartCountOper, testCase.RestartCount)) </ins> <del> By("it should get the expected 'Ready' status") Expect(status.Ready).To(Equal(testStatus.Ready)) </del> <ins> By("it should get the expected 'Phase'") Eventually(terminateContainer.GetPhase, retryTimeout, pollInterval).Should(Equal(testCase.Phase)) </ins> <del> By("it should get the expected 'State'") Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) </del> <ins> By("it should get the expected 'Ready' condition") Expect(terminateContainer.IsReady()).Should(Equal(testCase.Ready)) </ins> <del> By("it should be possible to delete [Conformance]") err = terminateContainer.Delete() Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { isPresent, err := terminateContainer.Present() return err == nil && !isPresent }, retryTimeout, pollInterval).Should(BeTrue()) } }) }) Context("when start a container that keeps running", func() { It("it should run with the expected status [Conformance]", func() { testContainer := api.Container{ Image:           ImageRegistry[busyBoxImage], Command:         []string{"sh", "-c", "while true; do echo hello; sleep 1; done"}, ImagePullPolicy: api.PullIfNotPresent, } testStatuses := []testStatus{ {"loop-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateRunning, "==", 0, true}, {"loop-cmd-rpof", api.RestartPolicyOnFailure, api.PodRunning, ContainerStateRunning, "==", 0, true}, {"loop-cmd-rpn", api.RestartPolicyNever, api.PodRunning, ContainerStateRunning, "==", 0, true}, } for _, testStatus := range testStatuses { testContainer.Name = testStatus.Name runningContainer := ConformanceContainer{ Container:     testContainer, Client:        cl, RestartPolicy: testStatus.RestartPolicy, NodeName:      *nodeName, Namespace:     namespace, } err := runningContainer.Create() Expect(err).NotTo(HaveOccurred()) testCContainers = append(testCContainers, runningContainer) Eventually(func() api.PodPhase { _, phase, _ := runningContainer.GetStatus() return phase }, retryTimeout, pollInterval).Should(Equal(api.PodRunning)) var status api.ContainerStatus var phase api.PodPhase Consistently(func() api.PodPhase { status, phase, err = runningContainer.GetStatus() return phase }, consistentCheckTimeout, pollInterval).Should(Equal(testStatus.Phase)) Expect(err).NotTo(HaveOccurred()) By("it should get the expected 'RestartCount'") Expect(status.RestartCount).To(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) By("it should get the expected 'Ready' status") Expect(status.Ready).To(Equal(testStatus.Ready)) </del> <ins> status, err := terminateContainer.GetStatus() Expect(err).ShouldNot(HaveOccurred()) </ins> By("it should get the expected 'State'") <del> Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) </del> <ins> Expect(GetContainerState(status.State)).To(Equal(testCase.State)) </ins> By("it should be possible to delete [Conformance]") <del> err = runningContainer.Delete() Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { isPresent, err := runningContainer.Present() return err == nil && !isPresent }, retryTimeout, pollInterval).Should(BeTrue()) } }) }) Context("when start a container that exits failure", func() { It("it should run with the expected status [Conformance]", func() { testContainer := api.Container{ Image:           ImageRegistry[busyBoxImage], Command:         []string{"false"}, ImagePullPolicy: api.PullIfNotPresent, } testStatuses := []testStatus{ {"fail-cmd-rpa", api.RestartPolicyAlways, api.PodRunning, ContainerStateWaiting | ContainerStateRunning | ContainerStateTerminated, ">", 0, false}, {"fail-cmd-rpof", api.RestartPolicyOnFailure, api.PodRunning, ContainerStateTerminated, ">", 0, false}, {"fail-cmd-rpn", api.RestartPolicyNever, api.PodFailed, ContainerStateTerminated, "==", 0, false}, } for _, testStatus := range testStatuses { testContainer.Name = testStatus.Name failureContainer := ConformanceContainer{ Container:     testContainer, Client:        cl, RestartPolicy: testStatus.RestartPolicy, NodeName:      *nodeName, Namespace:     namespace, } err := failureContainer.Create() Expect(err).NotTo(HaveOccurred()) testCContainers = append(testCContainers, failureContainer) Eventually(func() api.PodPhase { _, phase, _ := failureContainer.GetStatus() return phase }, retryTimeout, pollInterval).ShouldNot(Equal(api.PodPending)) var status api.ContainerStatus By("it should get the expected 'RestartCount'") Eventually(func() int32 { status, _, _ = failureContainer.GetStatus() return status.RestartCount }, retryTimeout, pollInterval).Should(BeNumerically(testStatus.RestartCountOper, testStatus.RestartCount)) By("it should get the expected 'Ready' status") Expect(status.Ready).To(Equal(testStatus.Ready)) By("it should get the expected 'State'") Expect(GetContainerState(status.State) & testStatus.State).NotTo(Equal(0)) By("it should be possible to delete [Conformance]") err = failureContainer.Delete() Expect(err).NotTo(HaveOccurred()) Eventually(func() bool { isPresent, err := failureContainer.Present() return err == nil && !isPresent }, retryTimeout, pollInterval).Should(BeTrue()) </del> <ins> Expect(terminateContainer.Delete()).To(Succeed()) Eventually(terminateContainer.Present, retryTimeout, pollInterval).Should(BeFalse()) </ins> } }) })