Change:
/kind bug What happened: We have noticed that one of the node-exporters is down. It turned out that service has "old" endpoint still. After digging deeper i see following errors in controller manager Turns out controller manager can not update endpoints for node-exporter service, because of this error. In endpoint list is see right IPs for worker nodes, but wrong IP for master node (that's why we detected the problem). Context: two worker nodes were recreated with new hostnames (generated randomly), but they use same ip address one master node were recreated with new hostname and new ip I've tried to manually update hostname and got same error, seems nodeName is not updatable field. It also affects all other endpoints, we were forced to recreated all pods (so they will change IPs) to be able to fix endpoints reconciliation. What you expected to happen: controller manager successfully update endpoints after node recreated with same IP, but different hostname. How to reproduce it (as minimally and precisely as possible): 1 master, 1 worker, service that points to pods with host network. k8s API service (keep it shut down) worker node with same IP and different host name to recreate pod on worker and in controller manager should appear error above. Anything else we need to know?: Environment: Kubernetes version (use ): 1.10.1 Cloud provider or hardware configuration: baremetal OS (e.g. from /etc/os-release): CoreOS 1688.5.3 Kernel (e.g. ): 4.14.32-coreos Install tools: own tool () Others:
/sig node
what is the use case of changing node name?
/sig api-machinery
We have ephemeral immutable nodes in our clusters. So upgrade or any other operation will cause node recreation. Hostname generated automatically, but the IP address depends on the underlay hypervisor node, where node will be scheduled. So it's possible that node stays on the same hypervisor and uses same ip address.
/cc
/assign
/cc do you think it's reasonable to loose endpoints validation?
/sig network this API is owned by sig network

Discussion:
return allErrs } <del> // Construct lookup map of old subset IPs to NodeNames. func updateEpAddrToNodeNameMap(ipToNodeName map[string]string, addresses []core.EndpointAddress) { for n := range addresses { if addresses[n].NodeName == nil { continue } ipToNodeName[addresses[n].IP] = *addresses[n].NodeName } } // Build a map across all subsets of IP -> NodeName func buildEndpointAddressNodeNameMap(subsets []core.EndpointSubset) map[string]string { ipToNodeName := make(map[string]string) for i := range subsets { updateEpAddrToNodeNameMap(ipToNodeName, subsets[i].Addresses) updateEpAddrToNodeNameMap(ipToNodeName, subsets[i].NotReadyAddresses) } return ipToNodeName } func validateEpAddrNodeNameTransition(addr *core.EndpointAddress, ipToNodeName map[string]string, fldPath *field.Path) field.ErrorList { errList := field.ErrorList{} existingNodeName, found := ipToNodeName[addr.IP] if !found { return errList } if addr.NodeName == nil || *addr.NodeName == existingNodeName { return errList } // NodeName entry found for this endpoint IP, but user is attempting to change NodeName return append(errList, field.Forbidden(fldPath, fmt.Sprintf("Cannot change NodeName for %s to %s", addr.IP, *addr.NodeName))) } </del> // ValidateEndpoints tests if required fields are set. func ValidateEndpoints(endpoints *core.Endpoints) field.ErrorList { allErrs := ValidateObjectMeta(&endpoints.ObjectMeta, true, ValidateEndpointsName, field.NewPath("metadata")) allErrs = append(allErrs, ValidateEndpointsSpecificAnnotations(endpoints.Annotations, field.NewPath("annotations"))...) <del> allErrs = append(allErrs, validateEndpointSubsets(endpoints.Subsets, []core.EndpointSubset{}, field.NewPath("subsets"))...) </del> <ins> allErrs = append(allErrs, validateEndpointSubsets(endpoints.Subsets, field.NewPath("subsets"))...) </ins> return allErrs } <del> func validateEndpointSubsets(subsets []core.EndpointSubset, oldSubsets []core.EndpointSubset, fldPath *field.Path) field.ErrorList { </del> <ins> func validateEndpointSubsets(subsets []core.EndpointSubset, fldPath *field.Path) field.ErrorList { </ins> allErrs := field.ErrorList{} <del> ipToNodeName := buildEndpointAddressNodeNameMap(oldSubsets) </del> for i := range subsets { ss := &subsets[i] idxPath := fldPath.Index(i)

allErrs = append(allErrs, field.Required(idxPath, "must specify `addresses` or `notReadyAddresses`")) } for addr := range ss.Addresses { <del> allErrs = append(allErrs, validateEndpointAddress(&ss.Addresses[addr], idxPath.Child("addresses").Index(addr), ipToNodeName)...) </del> <ins> allErrs = append(allErrs, validateEndpointAddress(&ss.Addresses[addr], idxPath.Child("addresses").Index(addr))...) </ins> } for addr := range ss.NotReadyAddresses { <del> allErrs = append(allErrs, validateEndpointAddress(&ss.NotReadyAddresses[addr], idxPath.Child("notReadyAddresses").Index(addr), ipToNodeName)...) </del> <ins> allErrs = append(allErrs, validateEndpointAddress(&ss.NotReadyAddresses[addr], idxPath.Child("notReadyAddresses").Index(addr))...) </ins> } for port := range ss.Ports { allErrs = append(allErrs, validateEndpointPort(&ss.Ports[port], len(ss.Ports) > 1, idxPath.Child("ports").Index(port))...)

return allErrs } <del> func validateEndpointAddress(address *core.EndpointAddress, fldPath *field.Path, ipToNodeName map[string]string) field.ErrorList { </del> <ins> func validateEndpointAddress(address *core.EndpointAddress, fldPath *field.Path) field.ErrorList { </ins> allErrs := field.ErrorList{} for _, msg := range validation.IsValidIP(address.IP) { allErrs = append(allErrs, field.Invalid(fldPath.Child("ip"), address.IP, msg))

allErrs = append(allErrs, field.Invalid(fldPath.Child("nodeName"), *address.NodeName, msg)) } } <del> allErrs = append(allErrs, validateEpAddrNodeNameTransition(address, ipToNodeName, fldPath.Child("nodeName"))...) if len(allErrs) > 0 { return allErrs } </del> allErrs = append(allErrs, validateNonSpecialIP(address.IP, fldPath.Child("ip"))...) return allErrs }

} // ValidateEndpointsUpdate tests to make sure an endpoints update can be applied. <ins> // NodeName changes are allowed during update to accommodate the case where nodeIP or PodCIDR is reused. // An existing endpoint ip will have a different nodeName if this happens. </ins> func ValidateEndpointsUpdate(newEndpoints, oldEndpoints *core.Endpoints) field.ErrorList { allErrs := ValidateObjectMetaUpdate(&newEndpoints.ObjectMeta, &oldEndpoints.ObjectMeta, field.NewPath("metadata")) <del> allErrs = append(allErrs, validateEndpointSubsets(newEndpoints.Subsets, oldEndpoints.Subsets, field.NewPath("subsets"))...) </del> <ins> allErrs = append(allErrs, validateEndpointSubsets(newEndpoints.Subsets, field.NewPath("subsets"))...) </ins> allErrs = append(allErrs, ValidateEndpointsSpecificAnnotations(newEndpoints.Annotations, field.NewPath("annotations"))...) return allErrs }

func TestEndpointAddressNodeNameUpdateRestrictions(t *testing.T) { oldEndpoint := newNodeNameEndpoint("kubernetes-node-setup-by-backend") updatedEndpoint := newNodeNameEndpoint("kubernetes-changed-nodename") <del> // Check that NodeName cannot be changed during update (if already set) </del> <ins> // Check that NodeName can be changed during update, this is to accommodate the case where nodeIP or PodCIDR is reused. // The same ip will now have a different nodeName. </ins> errList := ValidateEndpoints(updatedEndpoint) errList = append(errList, ValidateEndpointsUpdate(updatedEndpoint, oldEndpoint)...) <del> if len(errList) == 0 { t.Error("Endpoint should not allow changing of Subset.Addresses.NodeName on update") </del> <ins> if len(errList) != 0 { t.Error("Endpoint should allow changing of Subset.Addresses.NodeName on update") </ins> } }