Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --In a dual-stack cluster with IPv4 as default family (first kubelet IP), a container is created with a env var set to . When the container is running, properly contains the IPv4 But at runtime, this env var (used in container args) is expanded to the pod IPv6 instead of the IPv4. When using , it should always return the value actually exposed as and not something else. Kubernetes version:Cloud provider or hardware configuration: Bare metal cluster OS (e.g: ): Ubuntu 20.04 Kernel (e.g. ): Linux 5.8.0-59-generic 66~20.04.1-Ubuntu SMP Thu Jun 17 11:14:10 UTC 2021 x8664 x8664 x86_64 GNU/Linux Network plugin and version (if this is a network-related bug): cilium 1.10 Others: CRI-O 1.21 + crun.
Note: I have a configuration that matches the ones that had IP ordering issue before was fixed. So it may be related.
/sig network
Just a guess, but I wonder if it may be caused by
That's it. The fix from properly sort IP when computing the Pod status, but the sorted IPs are then replaced by computed IPs as this log shows.
/triage accepted Thanks for the detailed report
/assign

Discussion:
if err != nil { return "", err } <ins> // make podIPs order match node IP family preference #97979 podIPs = kl.sortPodIPs(podIPs) if len(podIPs) > 0 { podIP = podIPs[0] } </ins> switch internalFieldPath { case "spec.nodeName": return pod.Spec.NodeName, nil

return *s } <del> // convertStatusToAPIStatus creates an api PodStatus for the given pod from // the given internal pod status.  It is purely transformative and does not // alter the kubelet state at all. func (kl *Kubelet) convertStatusToAPIStatus(pod *v1.Pod, podStatus *kubecontainer.PodStatus) *v1.PodStatus { var apiPodStatus v1.PodStatus // The runtime pod status may have an arbitrary number of IPs, in an arbitrary // order. Pick out the first returned IP of the same IP family as the node IP // first, followed by the first IP of the opposite IP family (if any). podIPs := make([]v1.PodIP, 0, len(podStatus.IPs)) </del> <ins> // sortPodIPs return the PodIPs sorted and truncated by the cluster IP family preference. // The runtime pod status may have an arbitrary number of IPs, in an arbitrary order. // PodIPs are obtained by: func (m *kubeGenericRuntimeManager) determinePodSandboxIPs() // Pick out the first returned IP of the same IP family as the node IP // first, followed by the first IP of the opposite IP family (if any) // and use them for the Pod.Status.PodIPs and the Downward API environment variables func (kl *Kubelet) sortPodIPs(podIPs []string) []string { ips := make([]string, 0, 2) </ins> var validPrimaryIP, validSecondaryIP func(ip string) bool if len(kl.nodeIPs) == 0 || utilnet.IsIPv4(kl.nodeIPs[0]) { validPrimaryIP = utilnet.IsIPv4String

validPrimaryIP = utilnet.IsIPv6String validSecondaryIP = utilnet.IsIPv4String } <del> for _, ip := range podStatus.IPs { </del> <ins> for _, ip := range podIPs { </ins> if validPrimaryIP(ip) { <del> podIPs = append(podIPs, v1.PodIP{IP: ip}) </del> <ins> ips = append(ips, ip) </ins> break } } <del> for _, ip := range podStatus.IPs { </del> <ins> for _, ip := range podIPs { </ins> if validSecondaryIP(ip) { <del> podIPs = append(podIPs, v1.PodIP{IP: ip}) </del> <ins> ips = append(ips, ip) </ins> break } } <del> apiPodStatus.PodIPs = podIPs if len(podIPs) > 0 { apiPodStatus.PodIP = podIPs[0].IP </del> <ins> return ips } // convertStatusToAPIStatus creates an api PodStatus for the given pod from // the given internal pod status.  It is purely transformative and does not // alter the kubelet state at all. func (kl *Kubelet) convertStatusToAPIStatus(pod *v1.Pod, podStatus *kubecontainer.PodStatus) *v1.PodStatus { var apiPodStatus v1.PodStatus // copy pod status IPs to avoid race conditions with PodStatus #102806 podIPs := make([]string, len(podStatus.IPs)) for j, ip := range podStatus.IPs { podIPs[j] = ip } // make podIPs order match node IP family preference #97979 podIPs = kl.sortPodIPs(podIPs) for _, ip := range podIPs { apiPodStatus.PodIPs = append(apiPodStatus.PodIPs, v1.PodIP{IP: ip}) } if len(apiPodStatus.PodIPs) > 0 { apiPodStatus.PodIP = apiPodStatus.PodIPs[0].IP </ins> } // set status for Pods created on versions of kube older than 1.6

unsyncedServices   bool                   // whether the services should NOT be synced configMap          *v1.ConfigMap          // an optional ConfigMap to pull from secret             *v1.Secret             // an optional Secret to pull from <ins> podIPs             []string               // the pod IPs </ins> expectedEnvs       []kubecontainer.EnvVar // a set of expected environment vars expectedError      bool                   // does the test fail expectedEvent      string                 // does the test emit an event

}, }, }, <ins> podIPs:          []string{"1.2.3.4", "fd00::6"}, </ins> masterServiceNs: "nothing", nilLister:       true, expectedEnvs: []kubecontainer.EnvVar{

}, }, { <ins> name:               "downward api pod ips reverse order", ns:                 "downward-api", enableServiceLinks: &falseValue, container: &v1.Container{ Env: []v1.EnvVar{ { Name: "POD_IP", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ APIVersion: "v1", FieldPath:  "status.podIP", }, }, }, { Name: "POD_IPS", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ APIVersion: "v1", FieldPath:  "status.podIPs", }, }, }, { Name: "HOST_IP", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ APIVersion: "v1", FieldPath:  "status.hostIP", }, }, }, }, }, podIPs:          []string{"fd00::6", "1.2.3.4"}, masterServiceNs: "nothing", nilLister:       true, expectedEnvs: []kubecontainer.EnvVar{ {Name: "POD_IP", Value: "1.2.3.4"}, {Name: "POD_IPS", Value: "1.2.3.4,fd00::6"}, {Name: "HOST_IP", Value: testKubeletHostIP}, }, }, { name:               "downward api pod ips multiple ips", ns:                 "downward-api", enableServiceLinks: &falseValue, container: &v1.Container{ Env: []v1.EnvVar{ { Name: "POD_IP", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ APIVersion: "v1", FieldPath:  "status.podIP", }, }, }, { Name: "POD_IPS", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ APIVersion: "v1", FieldPath:  "status.podIPs", }, }, }, { Name: "HOST_IP", ValueFrom: &v1.EnvVarSource{ FieldRef: &v1.ObjectFieldSelector{ APIVersion: "v1", FieldPath:  "status.hostIP", }, }, }, }, }, podIPs:          []string{"1.2.3.4", "192.168.1.1.", "fd00::6"}, masterServiceNs: "nothing", nilLister:       true, expectedEnvs: []kubecontainer.EnvVar{ {Name: "POD_IP", Value: "1.2.3.4"}, {Name: "POD_IPS", Value: "1.2.3.4,fd00::6"}, {Name: "HOST_IP", Value: testKubeletHostIP}, }, }, { </ins> name:               "env expansion", ns:                 "test1", enableServiceLinks: &falseValue,

EnableServiceLinks: tc.enableServiceLinks, }, } <del> podIP := "1.2.3.4" podIPs := []string{"1.2.3.4,fd00::6"} </del> <ins> podIP := "" if len(tc.podIPs) > 0 { podIP = tc.podIPs[0] } </ins> if tc.staticPod { testPod.Annotations[kubetypes.ConfigSourceAnnotationKey] = "file" } <del> result, err := kl.makeEnvironmentVariables(testPod, tc.container, podIP, podIPs) </del> <ins> result, err := kl.makeEnvironmentVariables(testPod, tc.container, podIP, tc.podIPs) </ins> select { case e := <-fakeRecorder.Events: assert.Equal(t, tc.expectedEvent, e)

{IP: "192.168.0.1"}, }, }, <ins> { name: "CRI dual-stack PodIPs override NodeAddresses", nodeAddresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, {Type: v1.NodeInternalIP, Address: "fd01::1234"}, }, dualStack: true, criPodIPs: []string{"192.168.0.1", "2001:db8::2"}, podIPs: []v1.PodIP{ {IP: "192.168.0.1"}, {IP: "2001:db8::2"}, }, }, { // by default the cluster prefers IPv4 name: "CRI dual-stack PodIPs override NodeAddresses prefer IPv4", nodeAddresses: []v1.NodeAddress{ {Type: v1.NodeInternalIP, Address: "10.0.0.1"}, {Type: v1.NodeInternalIP, Address: "fd01::1234"}, }, dualStack: true, criPodIPs: []string{"2001:db8::2", "192.168.0.1"}, podIPs: []v1.PodIP{ {IP: "192.168.0.1"}, {IP: "2001:db8::2"}, }, }, </ins> } for _, tc := range testcases {

}) } } <ins> func TestSortPodIPs(t *testing.T) { testcases := []struct { name        string nodeIP      string podIPs      []string expectedIPs []string }{ { name:        "Simple", nodeIP:      "", podIPs:      []string{"10.0.0.1"}, expectedIPs: []string{"10.0.0.1"}, }, { name:        "Dual-stack", nodeIP:      "", podIPs:      []string{"10.0.0.1", "fd01::1234"}, expectedIPs: []string{"10.0.0.1", "fd01::1234"}, }, { name:        "Dual-stack with explicit node IP", nodeIP:      "192.168.1.1", podIPs:      []string{"10.0.0.1", "fd01::1234"}, expectedIPs: []string{"10.0.0.1", "fd01::1234"}, }, { name:        "Dual-stack with CRI returning wrong family first", nodeIP:      "", podIPs:      []string{"fd01::1234", "10.0.0.1"}, expectedIPs: []string{"10.0.0.1", "fd01::1234"}, }, { name:        "Dual-stack with explicit node IP with CRI returning wrong family first", nodeIP:      "192.168.1.1", podIPs:      []string{"fd01::1234", "10.0.0.1"}, expectedIPs: []string{"10.0.0.1", "fd01::1234"}, }, { name:        "Dual-stack with IPv6 node IP", nodeIP:      "fd00::5678", podIPs:      []string{"10.0.0.1", "fd01::1234"}, expectedIPs: []string{"fd01::1234", "10.0.0.1"}, }, { name:        "Dual-stack with IPv6 node IP, other CRI order", nodeIP:      "fd00::5678", podIPs:      []string{"fd01::1234", "10.0.0.1"}, expectedIPs: []string{"fd01::1234", "10.0.0.1"}, }, { name:        "No Pod IP matching Node IP", nodeIP:      "fd00::5678", podIPs:      []string{"10.0.0.1"}, expectedIPs: []string{"10.0.0.1"}, }, { name:        "No Pod IP matching (unspecified) Node IP", nodeIP:      "", podIPs:      []string{"fd01::1234"}, expectedIPs: []string{"fd01::1234"}, }, { name:        "Multiple IPv4 IPs", nodeIP:      "", podIPs:      []string{"10.0.0.1", "10.0.0.2", "10.0.0.3"}, expectedIPs: []string{"10.0.0.1"}, }, { name:        "Multiple Dual-Stack IPs", nodeIP:      "", podIPs:      []string{"10.0.0.1", "10.0.0.2", "fd01::1234", "10.0.0.3", "fd01::5678"}, expectedIPs: []string{"10.0.0.1", "fd01::1234"}, }, } for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */) defer testKubelet.Cleanup() kl := testKubelet.kubelet if tc.nodeIP != "" { kl.nodeIPs = []net.IP{net.ParseIP(tc.nodeIP)} } podIPs := kl.sortPodIPs(tc.podIPs) if !reflect.DeepEqual(podIPs, tc.expectedIPs) { t.Fatalf("Expected PodIPs %#v, got %#v", tc.expectedIPs, podIPs) } }) } } </ins>