Change:
<!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks! If the matter is security related, please disclose it privately via --What happened: ILB ip addresses come from the subnet that the K8s cluster uses. There is no ability to specify a different subnet. What you expected to happen: ILB ip addresses should be allocated from a different subnet if specified. How to reproduce it (as minimally and precisely as possible): Anything else we need to know?: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: GCE OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others:
/area platform/gce /assign
/sig network

Discussion:
g.clientBuilder = clientBuilder g.client = clientBuilder.ClientOrDie("cloud-provider") <del> if g.OnXPN() { g.eventBroadcaster = record.NewBroadcaster() g.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: g.client.CoreV1().Events("")}) g.eventRecorder = g.eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "g-cloudprovider"}) } </del> <ins> g.eventBroadcaster = record.NewBroadcaster() g.eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: g.client.CoreV1().Events("")}) g.eventRecorder = g.eventBroadcaster.NewRecorder(scheme.Scheme, v1.EventSource{Component: "g-cloudprovider"}) </ins> go g.watchClusterID(stop) }

// AlphaFeatureILBSubsets allows InternalLoadBalancer services to include a subset // of cluster nodes as backends instead of all nodes. AlphaFeatureILBSubsets = "ILBSubsets" <ins> // AlphaFeatureILBCustomSubnet allows InternalLoadBalancer services to specify a // network subnet to allocate ip addresses from. AlphaFeatureILBCustomSubnet = "ILBCustomSubnet" </ins> ) // AlphaFeatureGate contains a mapping of alpha features to whether they are enabled

// created in. ServiceAnnotationILBAllowGlobalAccess = "networking.gke.io/internal-load-balancer-allow-global-access" <ins> // ServiceAnnotationILBSubnet is annotated on a service with the name of the subnetwork // the ILB IP Address should be assigned from. By default, this is the subnetwork that the // cluster is created in. ServiceAnnotationILBSubnet = "networking.gke.io/internal-load-balancer-subnet" </ins> // NetworkTierAnnotationKey is annotated on a Service object to indicate which // network tier a GCP LB should use. The valid values are "Standard" and // "Premium" (default).

type ILBOptions struct { // AllowGlobalAccess Indicates whether global access is allowed for the LoadBalancer AllowGlobalAccess bool <ins> // SubnetName indicates which subnet the LoadBalancer VIPs should be assigned from SubnetName string </ins> } // GetLoadBalancerAnnotationAllowGlobalAccess returns if global access is enabled

func GetLoadBalancerAnnotationAllowGlobalAccess(service *v1.Service) bool { return service.Annotations[ServiceAnnotationILBAllowGlobalAccess] == "true" } <ins> // GetLoadBalancerAnnotationSubnet returns the configured subnet to assign LoadBalancer IP from. func GetLoadBalancerAnnotationSubnet(service *v1.Service) string { if val, exists := service.Annotations[ServiceAnnotationILBSubnet]; exists { return val } return "" } </ins>

g.eventRecorder.Event(svc, v1.EventTypeWarning, "ILBOptionsIgnored", "Internal LoadBalancer options are not supported with Legacy Networks.") options = ILBOptions{} } <ins> if !g.AlphaFeatureGate.Enabled(AlphaFeatureILBCustomSubnet) { if options.SubnetName != "" { g.eventRecorder.Event(svc, v1.EventTypeWarning, "ILBCustomSubnetOptionIgnored", "Internal LoadBalancer CustomSubnet options ignored as the feature gate is disabled.") options.SubnetName = "" } } </ins> loadBalancerName := g.GetLoadBalancerName(context.TODO(), clusterName, svc) sharedBackend := shareBackendService(svc)

return nil, err } <ins> subnetworkURL := g.SubnetworkURL() if g.AlphaFeatureGate.Enabled(AlphaFeatureILBCustomSubnet) { // If this feature is enabled, changes to subnet annotation will be // picked up and reflected in the forwarding rule. // Removing the annotation will set the forwarding rule to use the default subnet. if options.SubnetName != "" { subnetworkURL = gceSubnetworkURL("", g.networkProjectID, g.region, options.SubnetName) } } else { // TODO(84885) remove this once ILBCustomSubnet goes beta. if existingFwdRule != nil && existingFwdRule.Subnetwork != "" { // If the ILB already exists, continue using the subnet that it's already using. // This is to support existing ILBs that were setup using the wrong subnet - https://github.com/kubernetes/kubernetes/pull/57861 subnetworkURL = existingFwdRule.Subnetwork } } </ins> // Determine IP which will be used for this LB. If no forwarding rule has been established // or specified in the Service spec, then requestedIP = "". <del> requestedIP := determineRequestedIP(svc, existingFwdRule) ipToUse := requestedIP </del> <ins> ipToUse := ilbIPToUse(svc, existingFwdRule, subnetworkURL) </ins> <del> // If the ILB already exists, continue using the subnet that it's already using. // This is to support existing ILBs that were setup using the wrong subnet. subnetworkURL := g.SubnetworkURL() if existingFwdRule != nil && existingFwdRule.Subnetwork != "" { // external LBs have an empty Subnetwork field. subnetworkURL = existingFwdRule.Subnetwork } </del> <ins> klog.V(2).Infof("ensureInternalLoadBalancer(%v): Using subnet %s for LoadBalancer IP %s", loadBalancerName, options.SubnetName, ipToUse) </ins> var addrMgr *addressManager // If the network is not a legacy network, use the address manager if !g.IsLegacyNetwork() { <del> addrMgr = newAddressManager(g, nm.String(), g.Region(), subnetworkURL, loadBalancerName, requestedIP, cloud.SchemeInternal) </del> <ins> addrMgr = newAddressManager(g, nm.String(), g.Region(), subnetworkURL, loadBalancerName, ipToUse, cloud.SchemeInternal) </ins> ipToUse, err = addrMgr.HoldAddress() if err != nil { return nil, err

return fields[len(fields)-1] } <del> func determineRequestedIP(svc *v1.Service, fwdRule *compute.ForwardingRule) string { </del> <ins> // ilbIPToUse determines which IP address needs to be used in the ForwardingRule. If an IP has been // specified by the user, that is used. If there is an existing ForwardingRule, the ip address from // that is reused. In case a subnetwork change is requested, the existing ForwardingRule IP is ignored. func ilbIPToUse(svc *v1.Service, fwdRule *compute.ForwardingRule, requestedSubnet string) string { </ins> if svc.Spec.LoadBalancerIP != "" { return svc.Spec.LoadBalancerIP } <del> if fwdRule != nil { return fwdRule.IPAddress </del> <ins> if fwdRule == nil { return "" </ins> } <del> return "" </del> <ins> if requestedSubnet != fwdRule.Subnetwork { // reset ip address since subnet is being changed. return "" } return fwdRule.IPAddress </ins> } func getILBOptions(svc *v1.Service) ILBOptions { <del> return ILBOptions{AllowGlobalAccess: GetLoadBalancerAnnotationAllowGlobalAccess(svc)} </del> <ins> return ILBOptions{AllowGlobalAccess: GetLoadBalancerAnnotationAllowGlobalAccess(svc), SubnetName: GetLoadBalancerAnnotationSubnet(svc), } </ins> } // forwardingRuleComposite is a composite type encapsulating both the GA and Beta ForwardingRules.

f.lbScheme == other.lbScheme && equalStringSets(f.ports, other.ports) && f.backendService == other.backendService && <del> f.allowGlobalAccess == other.allowGlobalAccess </del> <ins> f.allowGlobalAccess == other.allowGlobalAccess && f.subnetwork == other.subnetwork </ins> } // toForwardingRuleComposite converts a compute beta or GA ForwardingRule into the composite type

t.Errorf("Expected frcGA and frcBeta rules to be unequal, got true") } } <ins> func TestEnsureInternalLoadBalancerCustomSubnet(t *testing.T) { t.Parallel() vals := DefaultTestClusterValues() gce, err := fakeGCECloud(vals) require.NoError(t, err) gce.AlphaFeatureGate = NewAlphaFeatureGate([]string{AlphaFeatureILBCustomSubnet}) nodeNames := []string{"test-node-1"} nodes, err := createAndInsertNodes(gce, nodeNames, vals.ZoneName) require.NoError(t, err) svc := fakeLoadbalancerService(string(LBTypeInternal)) status, err := createInternalLoadBalancer(gce, svc, nil, nodeNames, vals.ClusterName, vals.ClusterID, vals.ZoneName) lbName := gce.GetLoadBalancerName(context.TODO(), "", svc) if err != nil { t.Errorf("Unexpected error %v", err) } assert.NotEmpty(t, status.Ingress) fwdRule, err := gce.GetBetaRegionForwardingRule(lbName, gce.region) if err != nil || fwdRule == nil { t.Errorf("Unexpected error %v", err) } if fwdRule.Subnetwork != "" { t.Errorf("Unexpected subnet value %s in ILB ForwardingRule", fwdRule.Subnetwork) } // Change service to include the global access annotation and request static ip requestedIP := "4.5.6.7" svc.Annotations[ServiceAnnotationILBSubnet] = "test-subnet" svc.Spec.LoadBalancerIP = requestedIP status, err = gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, svc, nodes) if err != nil { t.Errorf("Unexpected error %v", err) } assert.NotEmpty(t, status.Ingress) if status.Ingress[0].IP != requestedIP { t.Errorf("Reserved IP %s not propagated, Got %s", requestedIP, status.Ingress[0].IP) } fwdRule, err = gce.GetBetaRegionForwardingRule(lbName, gce.region) if err != nil || fwdRule == nil { t.Errorf("Unexpected error %v", err) } if !strings.HasSuffix(fwdRule.Subnetwork, "test-subnet") { t.Errorf("Unexpected subnet value %s in ILB ForwardingRule.", fwdRule.Subnetwork) } // Change to a different subnet svc.Annotations[ServiceAnnotationILBSubnet] = "another-subnet" status, err = gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, svc, nodes) if err != nil { t.Errorf("Unexpected error %v", err) } assert.NotEmpty(t, status.Ingress) if status.Ingress[0].IP != requestedIP { t.Errorf("Reserved IP %s not propagated, Got %s", requestedIP, status.Ingress[0].IP) } fwdRule, err = gce.GetBetaRegionForwardingRule(lbName, gce.region) if err != nil || fwdRule == nil { t.Errorf("Unexpected error %v", err) } if !strings.HasSuffix(fwdRule.Subnetwork, "another-subnet") { t.Errorf("Unexpected subnet value %s in ILB ForwardingRule.", fwdRule.Subnetwork) } // remove the annotation - ILB should revert to default subnet. delete(svc.Annotations, ServiceAnnotationILBSubnet) status, err = gce.EnsureLoadBalancer(context.Background(), vals.ClusterName, svc, nodes) if err != nil { t.Errorf("Unexpected error %v", err) } assert.NotEmpty(t, status.Ingress) fwdRule, err = gce.GetBetaRegionForwardingRule(lbName, gce.region) if err != nil { t.Errorf("Unexpected error %v", err) } if fwdRule.Subnetwork != "" { t.Errorf("Unexpected subnet value %s in ILB ForwardingRule.", fwdRule.Subnetwork) } // Delete the service err = gce.EnsureLoadBalancerDeleted(context.Background(), vals.ClusterName, svc) if err != nil { t.Errorf("Unexpected error %v", err) } assertInternalLbResourcesDeleted(t, gce, svc, vals, true) } </ins>