Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: The NetworkPolicy e2e test fails regardless of which CNI plugin is enabled. The test assumes the client Pod can recheck connectivity after updating its label, but the client won't restart after the first failure, so the second check will always fail. What you expected to happen: The test should succeed if a CNI plugin implements NetworkPolicy correctly. How to reproduce it (as minimally and precisely as possible): a CNI that supports NetworkPolicy. the e2e test by focusing "should allow ingress access from updated pod". test will always fail. Anything else we need to know?: Environment: Kubernetes version (use ): 16.3 Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig testing /assign
/remove-sig testing /sig network
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry. ‍
/remove-triage unresolved
I see this in Calico 3.10 as well confirming that its definitely an issue.

Discussion:
defer cleanupNetworkPolicy(f, policy) ginkgo.By(fmt.Sprintf("Creating client pod %s that should not be able to connect to %s.", "client-a", service.Name)) <del> podClient := createNetworkClientPod(f, f.Namespace, "client-a", service, allowedPort) </del> <ins> // Specify RestartPolicy to OnFailure so we can check the client pod fails in the beginning and succeeds // after updating its label, otherwise it would not restart after the first failure. podClient := createNetworkClientPodWithRestartPolicy(f, f.Namespace, "client-a", service, allowedPort, v1.RestartPolicyOnFailure) </ins> defer func() { ginkgo.By(fmt.Sprintf("Cleaning up the pod %s", podClient.Name)) if err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Delete(context.TODO(), podClient.Name, nil); err != nil { framework.Failf("unable to cleanup pod %v: %v", podClient.Name, err) } }() <del> checkNoConnectivity(f, f.Namespace, podClient, service) </del> <ins> // Check Container exit code as restartable Pod's Phase will be Running even when container fails. checkNoConnectivityByExitCode(f, f.Namespace, podClient, service) </ins> ginkgo.By(fmt.Sprintf("Updating client pod %s that should successfully connect to %s.", podClient.Name, service.Name)) <del> podClient = updateNetworkClientPodLabel(f, f.Namespace, podClient.Name, "replace", "/metadata/labels", map[string]string{}) </del> <ins> podClient = updatePodLabel(f, f.Namespace, podClient.Name, "replace", "/metadata/labels", map[string]string{}) </ins> checkConnectivity(f, f.Namespace, podClient, service) }) <ins> ginkgo.It("should deny ingress access to updated pod [Feature:NetworkPolicy]", func() { const allowedPort = 80 ginkgo.By("Creating a network policy for the server which denies all traffic.") policy := &networkingv1.NetworkPolicy{ ObjectMeta: metav1.ObjectMeta{ Name: "deny-ingress-via-isolated-label-selector", }, Spec: networkingv1.NetworkPolicySpec{ PodSelector: metav1.LabelSelector{ MatchLabels: map[string]string{ "pod-name": podServerLabelSelector, }, MatchExpressions: []metav1.LabelSelectorRequirement{{ Key:      "isolated", Operator: metav1.LabelSelectorOpExists, }}, }, PolicyTypes: []networkingv1.PolicyType{networkingv1.PolicyTypeIngress}, Ingress:     []networkingv1.NetworkPolicyIngressRule{}, }, } policy, err := f.ClientSet.NetworkingV1().NetworkPolicies(f.Namespace.Name).Create(context.TODO(), policy, metav1.CreateOptions{}) framework.ExpectNoError(err, "Error creating Network Policy %v: %v", policy.ObjectMeta.Name, err) defer cleanupNetworkPolicy(f, policy) // Client can connect to service when the network policy doesn't apply to the server pod. testCanConnect(f, f.Namespace, "client-a", service, allowedPort) // Client cannot connect to service after updating the server pod's labels to match the network policy's selector. ginkgo.By(fmt.Sprintf("Updating server pod %s to be selected by network policy %s.", podServer.Name, policy.Name)) updatePodLabel(f, f.Namespace, podServer.Name, "add", "/metadata/labels/isolated", nil) testCannotConnect(f, f.Namespace, "client-a", service, allowedPort) }) </ins> ginkgo.It("should enforce egress policy allowing traffic to a server in a different namespace based on PodSelector and NamespaceSelector [Feature:NetworkPolicy]", func() { var nsBserviceA, nsBserviceB *v1.Service var nsBpodServerA, nsBpodServerB *v1.Pod

framework.Logf("Waiting for %s to complete.", podClient.Name) err = e2epod.WaitForPodSuccessInNamespace(f.ClientSet, podClient.Name, ns.Name) if err != nil { <del> // Collect pod logs when we see a failure. logs, logErr := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, podClient.Name, "client") if logErr != nil { framework.Failf("Error getting container logs: %s", logErr) } // Collect current NetworkPolicies applied in the test namespace. policies, err := f.ClientSet.NetworkingV1().NetworkPolicies(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{}) if err != nil { framework.Logf("error getting current NetworkPolicies for %s namespace: %s", f.Namespace.Name, err) } // Collect the list of pods running in the test namespace. podsInNS, err := e2epod.GetPodsInNamespace(f.ClientSet, f.Namespace.Name, map[string]string{}) if err != nil { framework.Logf("error getting pods for %s namespace: %s", f.Namespace.Name, err) } pods := []string{} for _, p := range podsInNS { pods = append(pods, fmt.Sprintf("Pod: %s, Status: %sn", p.Name, p.Status.String())) } </del> <ins> pods, policies, logs := collectPodsAndNetworkPolicies(f, podClient) </ins> framework.Failf("Pod %s should be able to connect to service %s, but was not able to connect.nPod logs:n%snn Current NetworkPolicies:nt%vnn Pods:nt%vnn", podClient.Name, service.Name, logs, policies.Items, pods) // Dump debug information for the test namespace.

// We expect an error here since it's a cannot connect test. // Dump debug information if the error was nil. if err == nil { <del> // Collect pod logs when we see a failure. logs, logErr := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, podClient.Name, "client") if logErr != nil { framework.Failf("Error getting container logs: %s", logErr) } </del> <ins> pods, policies, logs := collectPodsAndNetworkPolicies(f, podClient) framework.Failf("Pod %s should not be able to connect to service %s, but was able to connect.nPod logs:n%snn Current NetworkPolicies:nt%vnn Pods:nt %vnn", podClient.Name, service.Name, logs, policies.Items, pods) </ins> <del> // Collect current NetworkPolicies applied in the test namespace. policies, err := f.ClientSet.NetworkingV1().NetworkPolicies(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{}) if err != nil { framework.Logf("error getting current NetworkPolicies for %s namespace: %s", f.Namespace.Name, err) } </del> <ins> // Dump debug information for the test namespace. framework.DumpDebugInfo(f.ClientSet, f.Namespace.Name) } } </ins> <del> // Collect the list of pods running in the test namespace. podsInNS, err := e2epod.GetPodsInNamespace(f.ClientSet, f.Namespace.Name, map[string]string{}) if err != nil { framework.Logf("error getting pods for %s namespace: %s", f.Namespace.Name, err) </del> <ins> func checkNoConnectivityByExitCode(f *framework.Framework, ns *v1.Namespace, podClient *v1.Pod, service *v1.Service) { err := e2epod.WaitForPodCondition(f.ClientSet, ns.Name, podClient.Name, "terminated", framework.PodStartTimeout, func(pod *v1.Pod) (bool, error) { statuses := pod.Status.ContainerStatuses if len(statuses) == 0 || statuses[0].State.Terminated == nil { return false, nil </ins> } <del> pods := []string{} for _, p := range podsInNS { pods = append(pods, fmt.Sprintf("Pod: %s, Status: %sn", p.Name, p.Status.String())) </del> <ins> if statuses[0].State.Terminated.ExitCode != 0 { return true, fmt.Errorf("pod %q container exited with code: %d", podClient.Name, statuses[0].State.Terminated.ExitCode) </ins> } <del> framework.Failf("Pod %s should not be able to connect to service %s, but was able to connect.nPod logs:n%snn Current NetworkPolicies:nt%vnn Pods:nt %vnn", podClient.Name, service.Name, logs, policies.Items, pods) </del> <ins> return true, nil }) // We expect an error here since it's a cannot connect test. // Dump debug information if the error was nil. if err == nil { pods, policies, logs := collectPodsAndNetworkPolicies(f, podClient) framework.Failf("Pod %s should not be able to connect to service %s, but was able to connect.nPod logs:n%snn Current NetworkPolicies:nt%vnn Pods:nt%vnn", podClient.Name, service.Name, logs, policies.Items, pods) </ins> // Dump debug information for the test namespace. framework.DumpDebugInfo(f.ClientSet, f.Namespace.Name) } } <ins> func collectPodsAndNetworkPolicies(f *framework.Framework, podClient *v1.Pod) ([]string, *networkingv1.NetworkPolicyList, string) { // Collect pod logs when we see a failure. logs, logErr := e2epod.GetPodLogs(f.ClientSet, f.Namespace.Name, podClient.Name, "client") if logErr != nil { framework.Failf("Error getting container logs: %s", logErr) } // Collect current NetworkPolicies applied in the test namespace. policies, err := f.ClientSet.NetworkingV1().NetworkPolicies(f.Namespace.Name).List(context.TODO(), metav1.ListOptions{}) if err != nil { framework.Logf("error getting current NetworkPolicies for %s namespace: %s", f.Namespace.Name, err) } // Collect the list of pods running in the test namespace. podsInNS, err := e2epod.GetPodsInNamespace(f.ClientSet, f.Namespace.Name, map[string]string{}) if err != nil { framework.Logf("error getting pods for %s namespace: %s", f.Namespace.Name, err) } pods := []string{} for _, p := range podsInNS { pods = append(pods, fmt.Sprintf("Pod: %s, Status: %sn", p.Name, p.Status.String())) } return pods, policies, logs } </ins> // Create a server pod with a listening container for each port in ports[]. // Will also assign a pod label with key: "pod-name" and label set to the given podName for later use by the network // policy.

// This client will attempt a one-shot connection, then die, without restarting the pod. // Test can then be asserted based on whether the pod quit with an error or not. func createNetworkClientPod(f *framework.Framework, namespace *v1.Namespace, podName string, targetService *v1.Service, targetPort int) *v1.Pod { <ins> return createNetworkClientPodWithRestartPolicy(f, namespace, podName, targetService, targetPort, v1.RestartPolicyNever) } // Create a client pod which will attempt a netcat to the provided service, on the specified port. // It is similar to createNetworkClientPod but supports specifying RestartPolicy. func createNetworkClientPodWithRestartPolicy(f *framework.Framework, namespace *v1.Namespace, podName string, targetService *v1.Service, targetPort int, restartPolicy v1.RestartPolicy) *v1.Pod { </ins> pod, err := f.ClientSet.CoreV1().Pods(namespace.Name).Create(context.TODO(), &v1.Pod{ ObjectMeta: metav1.ObjectMeta{ GenerateName: podName + "-",

}, }, Spec: v1.PodSpec{ <del> RestartPolicy: v1.RestartPolicyNever, </del> <ins> RestartPolicy: restartPolicy, </ins> Containers: []v1.Container{ { Name:  "client",

return pod } <del> // Patch client pod with a map value func updateNetworkClientPodLabel(f *framework.Framework, namespace *v1.Namespace, podName string, patchOperation string, patchPath string, patchValue map[string]string) *v1.Pod { </del> <ins> // Patch pod with a map value func updatePodLabel(f *framework.Framework, namespace *v1.Namespace, podName string, patchOperation string, patchPath string, patchValue map[string]string) *v1.Pod { </ins> type patchMapValue struct { Op    string            `json:"op"` Path  string            `json:"path"`