Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --ipvs mode didn't preserve client IP for NodePort Service with protocol SCTP and ExternalTrafficPolicy Local. iptables mode works fine. ipvs mode with TCP and UDP protocol works fine. Client IP for NodePort Service should be preseved when ExternalTrafficPolicy is Local regardless of the mode and the protocol. kube-proxy mode to ipvs a NodePort service with protocol SCTP and ExternalTrafficPolicy Local the NodePort service, the client IP was not preserved This is caused by wrong order of iptables rules that match kubeNodePortLocalSetSCTP and kubeNodePortSetSCTP: Kubernetes version (use ): v1.22.0 Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/sig network /assign
/triage accepted

Discussion:
{kubeNodePortSetTCP, string(KubeNodePortChain), string(KubeMarkMasqChain), "dst", utilipset.ProtocolTCP}, {kubeNodePortLocalSetUDP, string(KubeNodePortChain), "RETURN", "dst", utilipset.ProtocolUDP}, {kubeNodePortSetUDP, string(KubeNodePortChain), string(KubeMarkMasqChain), "dst", utilipset.ProtocolUDP}, <del> {kubeNodePortSetSCTP, string(KubeNodePortChain), string(KubeMarkMasqChain), "dst,dst", utilipset.ProtocolSCTP}, </del> {kubeNodePortLocalSetSCTP, string(KubeNodePortChain), "RETURN", "dst,dst", utilipset.ProtocolSCTP}, <ins> {kubeNodePortSetSCTP, string(KubeNodePortChain), string(KubeMarkMasqChain), "dst,dst", utilipset.ProtocolSCTP}, </ins> } // In IPVS proxy mode, the following flags need to be set

expectedIptablesChains: netlinktest.ExpectedIptablesChain{ string(KubeNodePortChain): {{ JumpChain: string(KubeMarkMasqChain), MatchSet: kubeNodePortSetUDP, <ins> }, { JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet, </ins> }}, string(kubeServicesChain): {{ <ins> JumpChain: string(KubeMarkMasqChain), MatchSet: kubeClusterIPSet, }, { </ins> JumpChain: string(KubeNodePortChain), MatchSet: "", <ins> }, { JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, </ins> }}, }, },

}, }, }, <ins> { name: "node port service with protocol sctp and externalTrafficPolicy local", services: []*v1.Service{ makeTestService("ns1", "svc1", func(svc *v1.Service) { svc.Spec.Type = "NodePort" svc.Spec.ClusterIP = "10.20.30.41" svc.Spec.Ports = []v1.ServicePort{{ Name:     "p80", Port:     int32(80), Protocol: v1.ProtocolSCTP, NodePort: int32(3001), }} svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal }), }, endpoints: []*discovery.EndpointSlice{ makeTestEndpointSlice("ns1", "svc1", 1, func(eps *discovery.EndpointSlice) { eps.AddressType = discovery.AddressTypeIPv4 eps.Endpoints = []discovery.Endpoint{{ Addresses: []string{"10.180.0.1"}, NodeName:  utilpointer.StringPtr(testHostname), }, { Addresses: []string{"10.180.1.1"}, NodeName:  utilpointer.StringPtr("otherHost"), }} eps.Ports = []discovery.EndpointPort{{ Name:     utilpointer.String("p80"), Port:     utilpointer.Int32(80), Protocol: &sctpProtocol, }} }), }, nodeIPs: []net.IP{ netutils.ParseIPSloppy("100.101.102.103"), }, nodePortAddresses: []string{}, expectedIPVS: &ipvstest.FakeIPVS{ Services: map[ipvstest.ServiceKey]*utilipvs.VirtualServer{ { IP:       "10.20.30.41", Port:     80, Protocol: "SCTP", }: { Address:   netutils.ParseIPSloppy("10.20.30.41"), Protocol:  "SCTP", Port:      uint16(80), Scheduler: "rr", }, { IP:       "100.101.102.103", Port:     3001, Protocol: "SCTP", }: { Address:   netutils.ParseIPSloppy("100.101.102.103"), Protocol:  "SCTP", Port:      uint16(3001), Scheduler: "rr", }, }, Destinations: map[ipvstest.ServiceKey][]*utilipvs.RealServer{ { IP:       "10.20.30.41", Port:     80, Protocol: "SCTP", }: { { Address: netutils.ParseIPSloppy("10.180.0.1"), Port:    uint16(80), Weight:  1, }, { Address: netutils.ParseIPSloppy("10.180.1.1"), Port:    uint16(80), Weight:  1, }, }, { IP:       "100.101.102.103", Port:     3001, Protocol: "SCTP", }: { { Address: netutils.ParseIPSloppy("10.180.0.1"), Port:    uint16(80), Weight:  1, }, }, }, }, expectedIPSets: netlinktest.ExpectedIPSet{ kubeNodePortSetSCTP: { { IP:       "100.101.102.103", Port:     3001, Protocol: strings.ToLower(string(v1.ProtocolSCTP)), SetType:  utilipset.HashIPPort, }, }, kubeNodePortLocalSetSCTP: { { IP:       "100.101.102.103", Port:     3001, Protocol: strings.ToLower(string(v1.ProtocolSCTP)), SetType:  utilipset.HashIPPort, }, }, }, expectedIptablesChains: netlinktest.ExpectedIptablesChain{ string(KubeNodePortChain): {{ JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetSCTP, }, { JumpChain: string(KubeMarkMasqChain), MatchSet: kubeNodePortSetSCTP, }, { JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet, }}, }, }, </ins> } for _, test := range tests {

epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ JumpChain: string(KubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet, <ins> }, { JumpChain: string(KubeMarkMasqChain), MatchSet: kubeClusterIPSet, }, { JumpChain: string(KubeNodePortChain), MatchSet: "", }, { JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, }, { JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet, </ins> }}, string(kubeLoadBalancerSet): {{ JumpChain: string(KubeMarkMasqChain), MatchSet: "",

// Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ <ins> JumpChain: string(KubeMarkMasqChain), MatchSet: kubeClusterIPSet, }, { </ins> JumpChain: string(KubeNodePortChain), MatchSet: "", <ins> }, { JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, </ins> }}, string(KubeNodePortChain): {{ JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetTCP, }, { JumpChain: string(KubeMarkMasqChain), MatchSet: kubeNodePortSetTCP, <ins> }, { JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet, </ins> }}, } checkIptables(t, ipt, epIpt)

// Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(KubeNodePortChain): {{ <ins> JumpChain: "RETURN", MatchSet: kubeNodePortLocalSetTCP, }, { JumpChain: string(KubeMarkMasqChain), MatchSet: kubeNodePortSetTCP, }, { </ins> JumpChain: "ACCEPT", MatchSet: kubeHealthCheckNodePortSet, }}, }

epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ JumpChain: string(KubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet, <ins> }, { JumpChain: string(KubeMarkMasqChain), MatchSet: kubeClusterIPSet, }, { JumpChain: string(KubeNodePortChain), MatchSet: "", }, { JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, }, { JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet, </ins> }}, string(KubeLoadBalancerChain): {{ JumpChain: string(KubeFireWallChain), MatchSet: kubeLoadbalancerFWSet,

// Check iptables chain and rules epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): { <ins> {JumpChain: string(KubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet}, {JumpChain: string(KubeMarkMasqChain), MatchSet: kubeClusterIPSet}, {JumpChain: string(KubeMarkMasqChain), MatchSet: kubeExternalIPSet}, {JumpChain: "ACCEPT", MatchSet: kubeExternalIPSet}, // With externalTrafficOnlyArgs {JumpChain: "ACCEPT", MatchSet: kubeExternalIPSet}, // With dstLocalOnlyArgs {JumpChain: string(KubeNodePortChain), MatchSet: ""}, </ins> {JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet}, {JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet}, <del> {JumpChain: "ACCEPT", MatchSet: kubeExternalIPSet}, </del> }, } checkIptables(t, ipt, epIpt)

epIpt := netlinktest.ExpectedIptablesChain{ string(kubeServicesChain): {{ JumpChain: string(KubeLoadBalancerChain), MatchSet: kubeLoadBalancerSet, <ins> }, { JumpChain: string(KubeMarkMasqChain), MatchSet: kubeClusterIPSet, }, { JumpChain: string(KubeNodePortChain), MatchSet: "", }, { JumpChain: "ACCEPT", MatchSet: kubeClusterIPSet, }, { JumpChain: "ACCEPT", MatchSet: kubeLoadBalancerSet, </ins> }}, string(KubeLoadBalancerChain): {{ JumpChain: "RETURN", MatchSet: kubeLoadBalancerLocalSet,

return false } <del> // checkIptabless to check expected iptables chain and rules </del> <ins> // checkIptables to check expected iptables chain and rules. The got rules must have same number and order as the // expected rules. </ins> func checkIptables(t *testing.T, ipt *iptablestest.FakeIPTables, epIpt netlinktest.ExpectedIptablesChain) { for epChain, epRules := range epIpt { rules := ipt.GetRules(epChain) <del> for _, epRule := range epRules { if !hasJump(rules, epRule.JumpChain, epRule.MatchSet) { t.Errorf("Didn't find jump from chain %v match set %v to %v", epChain, epRule.MatchSet, epRule.JumpChain) </del> <ins> if len(rules) != len(epRules) { t.Errorf("Expected %d iptables rule in chain %s, got %d", len(epRules), epChain, len(rules)) continue } for i, epRule := range epRules { rule := rules[i] if rule[iptablestest.Jump] != epRule.JumpChain || !strings.Contains(rule[iptablestest.MatchSet], epRule.MatchSet) { t.Errorf("Expected MatchSet=%s JumpChain=%s, got MatchSet=%s JumpChain=%s", epRule.MatchSet, epRule.JumpChain, rule[iptablestest.MatchSet], rule[iptablestest.Jump]) </ins> } } }