Change:
<!-- This form is for bug reports and feature requests ONLY! If you're looking for help check and the . If the matter is security related, please disclose it privately via --Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: Table-based e2e tests are not getting updated to leading to the following error- ( mentioned in ) framework.ConformanceIt() requires the first parameter to be and are associated with at runtime based on different cases/test-scenarios. What you expected to happen: Either disapprove the use of table-based e2e tests or suggestion on an alternative approach to add such tests to Conformance. How to reproduce it (as minimally and precisely as possible): framework.ConformanceIt() to . /area conformance cc
/sig testing
/cc
Approaches to work around this: wrap the whole table based test in a ConformanceIt block refactor to a more boilerplate approach that has a ConformanceIt block for each table entry update the conformance test walker to detect table based tests that use a well known field?
Thanks I am evaluating these approaches especially on "conformance test walker" to retrieve well-known fields like and variables used inside .
Does this close if closes or are there other tests that need to transition?
This issue would be No other e2e test needs the transition from table-based test structure. Though "only one" remaining test should be restructured as an individual test.
/close Closing as per other fruitful approach and efforts would be taken to refactor table-based tests into individual tests wherever needed.
Closing this issue. <detailsIn response to : /close Closing as per other fruitful approach and efforts would be taken to refactor table-based tests into individual tests wherever needed. Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details

Discussion:
import ( "fmt" "path" <del> "strings" </del> "time" "k8s.io/api/core/v1"

Eventually(terminateContainer.Present, ContainerStatusRetryTimeout, ContainerStatusPollInterval).Should(BeFalse()) } }) <ins> }) Context("on terminated container", func() { </ins> rootUser := int64(0) nonRootUser := int64(10000) <del> for _, testCase := range []struct { name      string container v1.Container phase     v1.PodPhase message   gomegatypes.GomegaMatcher }{ { name: "if TerminationMessagePath is set [NodeConformance]", container: v1.Container{ Image:                  framework.BusyBoxImage, Command:                []string{"/bin/sh", "-c"}, Args:                   []string{"/bin/echo -n DONE > /dev/termination-log"}, TerminationMessagePath: "/dev/termination-log", SecurityContext: &v1.SecurityContext{ RunAsUser: &rootUser, }, }, phase:   v1.PodSucceeded, message: Equal("DONE"), }, { name: "if TerminationMessagePath is set as non-root user and at a non-default path [NodeConformance]", container: v1.Container{ Image:                  framework.BusyBoxImage, Command:                []string{"/bin/sh", "-c"}, Args:                   []string{"/bin/echo -n DONE > /dev/termination-custom-log"}, TerminationMessagePath: "/dev/termination-custom-log", SecurityContext: &v1.SecurityContext{ RunAsUser: &nonRootUser, }, }, phase:   v1.PodSucceeded, message: Equal("DONE"), }, { name: "from log output if TerminationMessagePolicy FallbackToLogOnError is set [NodeConformance]", container: v1.Container{ Image:                    framework.BusyBoxImage, Command:                  []string{"/bin/sh", "-c"}, Args:                     []string{"/bin/echo -n DONE; /bin/false"}, TerminationMessagePath:   "/dev/termination-log", TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, }, phase:   v1.PodFailed, message: Equal("DONE"), }, { name: "as empty when pod succeeds and TerminationMessagePolicy FallbackToLogOnError is set [NodeConformance]", container: v1.Container{ Image:                    framework.BusyBoxImage, Command:                  []string{"/bin/sh", "-c"}, Args:                     []string{"/bin/echo DONE; /bin/true"}, TerminationMessagePath:   "/dev/termination-log", TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, }, phase:   v1.PodSucceeded, message: Equal(""), }, { name: "from file when pod succeeds and TerminationMessagePolicy FallbackToLogOnError is set [NodeConformance]", container: v1.Container{ Image:                    framework.BusyBoxImage, Command:                  []string{"/bin/sh", "-c"}, Args:                     []string{"/bin/echo -n OK > /dev/termination-log; /bin/echo DONE; /bin/true"}, TerminationMessagePath:   "/dev/termination-log", TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, }, phase:   v1.PodSucceeded, message: Equal("OK"), }, } { testCase := testCase It(fmt.Sprintf("should report termination message %s [LinuxOnly]", testCase.name), func() { // Cannot mount files in Windows Containers. testCase.container.Name = "termination-message-container" c := ConformanceContainer{ PodClient:     f.PodClient(), Container:     testCase.container, RestartPolicy: v1.RestartPolicyNever, } By("create the container") c.Create() defer c.Delete() </del> <ins> // Create and then terminate the container under defined PodPhase to verify if termination message matches the expected output. Lastly delete the created container. matchTerminationMessage := func(container v1.Container, expectedPhase v1.PodPhase, expectedMsg gomegatypes.GomegaMatcher) { container.Name = "termination-message-container" c := ConformanceContainer{ PodClient:     f.PodClient(), Container:     container, RestartPolicy: v1.RestartPolicyNever, } </ins> <del> By(fmt.Sprintf("wait for the container to reach %s", testCase.phase)) Eventually(c.GetPhase, ContainerStatusRetryTimeout, ContainerStatusPollInterval).Should(Equal(testCase.phase)) </del> <ins> By("create the container") c.Create() defer c.Delete() </ins> <del> By("get the container status") status, err := c.GetStatus() Expect(err).NotTo(HaveOccurred()) </del> <ins> By(fmt.Sprintf("wait for the container to reach %s", expectedPhase)) Eventually(c.GetPhase, ContainerStatusRetryTimeout, ContainerStatusPollInterval).Should(Equal(expectedPhase)) </ins> <del> By("the container should be terminated") Expect(GetContainerState(status.State)).To(Equal(ContainerStateTerminated)) </del> <ins> By("get the container status") status, err := c.GetStatus() Expect(err).NotTo(HaveOccurred()) </ins> <del> By("the termination message should be set") Expect(status.State.Terminated.Message).Should(testCase.message) </del> <ins> By("the container should be terminated") Expect(GetContainerState(status.State)).To(Equal(ContainerStateTerminated)) </ins> <del> By("delete the container") Expect(c.Delete()).To(Succeed()) }) </del> <ins> By("the termination message should be set") framework.Logf("Expected: %v to match Container's Termination Message: %v --", expectedMsg, status.State.Terminated.Message) Expect(status.State.Terminated.Message).Should(expectedMsg) By("delete the container") Expect(c.Delete()).To(Succeed()) </ins> } <ins> It("should report termination message [LinuxOnly] if TerminationMessagePath is set [NodeConformance]", func() { // Cannot mount files in Windows Containers. container := v1.Container{ Image:                  framework.BusyBoxImage, Command:                []string{"/bin/sh", "-c"}, Args:                   []string{"/bin/echo -n DONE > /dev/termination-log"}, TerminationMessagePath: "/dev/termination-log", SecurityContext: &v1.SecurityContext{ RunAsUser: &rootUser, }, } matchTerminationMessage(container, v1.PodSucceeded, Equal("DONE")) }) It("should report termination message [LinuxOnly] if TerminationMessagePath is set as non-root user and at a non-default path [NodeConformance]", func() { // Cannot mount files in Windows Containers. container := v1.Container{ Image:                  framework.BusyBoxImage, Command:                []string{"/bin/sh", "-c"}, Args:                   []string{"/bin/echo -n DONE > /dev/termination-custom-log"}, TerminationMessagePath: "/dev/termination-custom-log", SecurityContext: &v1.SecurityContext{ RunAsUser: &nonRootUser, }, } matchTerminationMessage(container, v1.PodSucceeded, Equal("DONE")) }) It("should report termination message [LinuxOnly] from log output if TerminationMessagePolicy FallbackToLogsOnError is set [NodeConformance]", func() { // Cannot mount files in Windows Containers. container := v1.Container{ Image:                    framework.BusyBoxImage, Command:                  []string{"/bin/sh", "-c"}, Args:                     []string{"/bin/echo -n DONE; /bin/false"}, TerminationMessagePath:   "/dev/termination-log", TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, } matchTerminationMessage(container, v1.PodFailed, Equal("DONE")) }) It("should report termination message [LinuxOnly] as empty when pod succeeds and TerminationMessagePolicy FallbackToLogsOnError is set [NodeConformance]", func() { // Cannot mount files in Windows Containers. container := v1.Container{ Image:                    framework.BusyBoxImage, Command:                  []string{"/bin/sh", "-c"}, Args:                     []string{"/bin/echo DONE; /bin/true"}, TerminationMessagePath:   "/dev/termination-log", TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, } matchTerminationMessage(container, v1.PodSucceeded, Equal("")) }) It("should report termination message [LinuxOnly] from file when pod succeeds and TerminationMessagePolicy FallbackToLogsOnError is set [NodeConformance]", func() { // Cannot mount files in Windows Containers. container := v1.Container{ Image:                    framework.BusyBoxImage, Command:                  []string{"/bin/sh", "-c"}, Args:                     []string{"/bin/echo -n OK > /dev/termination-log; /bin/echo DONE; /bin/true"}, TerminationMessagePath:   "/dev/termination-log", TerminationMessagePolicy: v1.TerminationMessageFallbackToLogsOnError, } matchTerminationMessage(container, v1.PodSucceeded, Equal("OK")) }) </ins> }) Context("when running a container with a new image", func() { <del> // The service account only has pull permission auth := ` </del> <ins> // Images used for ConformanceContainer are not added into NodeImageWhiteList, because this test is // testing image pulling, these images don't need to be prepulled. The ImagePullPolicy // is v1.PullAlways, so it won't be blocked by framework image white list check. imagePullTest := func(image string, hasSecret bool, expectedPhase v1.PodPhase, expectedPullStatus bool) { container := ConformanceContainer{ PodClient: f.PodClient(), Container: v1.Container{ Name:            "image-pull-test", Image:           image, Command:         []string{"/bin/sh", "-c", "while true; do sleep 1; done"}, ImagePullPolicy: v1.PullAlways, }, RestartPolicy: v1.RestartPolicyNever, } if hasSecret { // The service account only has pull permission auth := ` </ins> { "auths": { "https://gcr.io": {

} } }` <del> secret := &v1.Secret{ Data: map[string][]byte{v1.DockerConfigJsonKey: []byte(auth)}, Type: v1.SecretTypeDockerConfigJson, } // The following images are not added into NodeImageWhiteList, because this test is // testing image pulling, these images don't need to be prepulled. The ImagePullPolicy // is v1.PullAlways, so it won't be blocked by framework image white list check. for _, testCase := range []struct { description string image       string secret      bool phase       v1.PodPhase waiting     bool }{ { description: "should not be able to pull image from invalid registry", image:       "invalid.com/invalid/alpine:3.1", phase:       v1.PodPending, waiting:     true, }, { description: "should not be able to pull non-existing image from gcr.io", image:       "k8s.gcr.io/invalid-image:invalid-tag", phase:       v1.PodPending, waiting:     true, }, { // TODO(claudiub): Add a Windows equivalent test. description: "should be able to pull image from gcr.io [LinuxOnly]", image:       "gcr.io/google-containers/debian-base:0.4.1", phase:       v1.PodRunning, waiting:     false, }, { description: "should be able to pull image from docker hub [LinuxOnly]", image:       "alpine:3.7", phase:       v1.PodRunning, waiting:     false, }, { description: "should be able to pull image from docker hub [WindowsOnly]", image:       "e2eteam/busybox:1.29", phase:       v1.PodRunning, waiting:     false, }, { description: "should not be able to pull from private registry without secret", image:       "gcr.io/authenticated-image-pulling/alpine:3.7", phase:       v1.PodPending, waiting:     true, }, { description: "should be able to pull from private registry with secret [LinuxOnly]", image:       "gcr.io/authenticated-image-pulling/alpine:3.7", secret:      true, phase:       v1.PodRunning, waiting:     false, }, } { testCase := testCase It(testCase.description+" [NodeConformance]", func() { if strings.Contains(testCase.description, "[WindowsOnly]") { framework.SkipUnlessNodeOSDistroIs("windows") } name := "image-pull-test" command := []string{"/bin/sh", "-c", "while true; do sleep 1; done"} container := ConformanceContainer{ PodClient: f.PodClient(), Container: v1.Container{ Name:    name, Image:   testCase.image, Command: command, // PullAlways makes sure that the image will always be pulled even if it is present before the test. ImagePullPolicy: v1.PullAlways, }, RestartPolicy: v1.RestartPolicyNever, </del> <ins> secret := &v1.Secret{ Data: map[string][]byte{v1.DockerConfigJsonKey: []byte(auth)}, Type: v1.SecretTypeDockerConfigJson, </ins> } <del> if testCase.secret { secret.Name = "image-pull-secret-" + string(uuid.NewUUID()) By("create image pull secret") _, err := f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret) Expect(err).NotTo(HaveOccurred()) defer f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Delete(secret.Name, nil) container.ImagePullSecrets = []string{secret.Name} </del> <ins> secret.Name = "image-pull-secret-" + string(uuid.NewUUID()) By("create image pull secret") _, err := f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Create(secret) Expect(err).NotTo(HaveOccurred()) defer f.ClientSet.CoreV1().Secrets(f.Namespace.Name).Delete(secret.Name, nil) container.ImagePullSecrets = []string{secret.Name} } // checkContainerStatus checks whether the container status matches expectation. checkContainerStatus := func() error { status, err := container.GetStatus() if err != nil { return fmt.Errorf("failed to get container status: %v", err) </ins> } <del> // checkContainerStatus checks whether the container status matches expectation. checkContainerStatus := func() error { status, err := container.GetStatus() if err != nil { return fmt.Errorf("failed to get container status: %v", err) } // We need to check container state first. The default pod status is pending, If we check // pod phase first, and the expected pod phase is Pending, the container status may not // even show up when we check it. // Check container state if !testCase.waiting { if status.State.Running == nil { return fmt.Errorf("expected container state: Running, got: %q", GetContainerState(status.State)) } </del> <ins> // We need to check container state first. The default pod status is pending, If we check pod phase first, // and the expected pod phase is Pending, the container status may not even show up when we check it. // Check container state if !expectedPullStatus { if status.State.Running == nil { return fmt.Errorf("expected container state: Running, got: %q", GetContainerState(status.State)) </ins> } <del> if testCase.waiting { if status.State.Waiting == nil { return fmt.Errorf("expected container state: Waiting, got: %q", GetContainerState(status.State)) } reason := status.State.Waiting.Reason if reason != images.ErrImagePull.Error() && reason != images.ErrImagePullBackOff.Error() { return fmt.Errorf("unexpected waiting reason: %q", reason) } } // Check pod phase phase, err := container.GetPhase() if err != nil { return fmt.Errorf("failed to get pod phase: %v", err) </del> <ins> } if expectedPullStatus { if status.State.Waiting == nil { return fmt.Errorf("expected container state: Waiting, got: %q", GetContainerState(status.State)) </ins> } <del> if phase != testCase.phase { return fmt.Errorf("expected pod phase: %q, got: %q", testCase.phase, phase) </del> <ins> reason := status.State.Waiting.Reason if reason != images.ErrImagePull.Error() && reason != images.ErrImagePullBackOff.Error() { return fmt.Errorf("unexpected waiting reason: %q", reason) </ins> } <del> return nil </del> } <del> // The image registry is not stable, which sometimes causes the test to fail. Add retry mechanism to make this // less flaky. const flakeRetry = 3 for i := 1; i <= flakeRetry; i++ { var err error By("create the container") container.Create() By("check the container status") for start := time.Now(); time.Since(start) < ContainerStatusRetryTimeout; time.Sleep(ContainerStatusPollInterval) { if err = checkContainerStatus(); err == nil { break } } By("delete the container") container.Delete() if err == nil { </del> <ins> // Check pod phase phase, err := container.GetPhase() if err != nil { return fmt.Errorf("failed to get pod phase: %v", err) } if phase != expectedPhase { return fmt.Errorf("expected pod phase: %q, got: %q", expectedPhase, phase) } return nil } // The image registry is not stable, which sometimes causes the test to fail. Add retry mechanism to make this less flaky. const flakeRetry = 3 for i := 1; i <= flakeRetry; i++ { var err error By("create the container") container.Create() By("check the container status") for start := time.Now(); time.Since(start) < ContainerStatusRetryTimeout; time.Sleep(ContainerStatusPollInterval) { if err = checkContainerStatus(); err == nil { </ins> break } <del> if i < flakeRetry { framework.Logf("No.%d attempt failed: %v, retrying...", i, err) } else { framework.Failf("All %d attempts failed: %v", flakeRetry, err) } </del> } <del> }) </del> <ins> By("delete the container") container.Delete() if err == nil { break } if i < flakeRetry { framework.Logf("No.%d attempt failed: %v, retrying...", i, err) } else { framework.Failf("All %d attempts failed: %v", flakeRetry, err) } } </ins> } <ins> It("should not be able to pull image from invalid registry [NodeConformance]", func() { image := "invalid.com/invalid/alpine:3.1" imagePullTest(image, false, v1.PodPending, true) }) It("should not be able to pull non-existing image from gcr.io [NodeConformance]", func() { image := "k8s.gcr.io/invalid-image:invalid-tag" imagePullTest(image, false, v1.PodPending, true) }) // TODO(claudiub): Add a Windows equivalent test. It("should be able to pull image from gcr.io [LinuxOnly] [NodeConformance]", func() { image := "gcr.io/google-containers/debian-base:0.4.1" imagePullTest(image, false, v1.PodRunning, false) }) It("should be able to pull image from docker hub [LinuxOnly] [NodeConformance]", func() { image := "alpine:3.7" imagePullTest(image, false, v1.PodRunning, false) }) It("should be able to pull image from docker hub [WindowsOnly] [NodeConformance]", func() { framework.SkipUnlessNodeOSDistroIs("windows") image := "e2eteam/busybox:1.29" imagePullTest(image, false, v1.PodRunning, false) }) It("should not be able to pull from private registry without secret [NodeConformance]", func() { image := "gcr.io/authenticated-image-pulling/alpine:3.7" imagePullTest(image, false, v1.PodPending, true) }) It("should be able to pull from private registry with secret [LinuxOnly] [NodeConformance]", func() { image := "gcr.io/authenticated-image-pulling/alpine:3.7" imagePullTest(image, true, v1.PodRunning, false) }) </ins> }) }) })