Change:
What happened: kube-controller-manager panic for What you expected to happen: no panic. How to reproduce it (as minimally and precisely as possible): No idea. I guess it's hard to reproduce. Anything else we need to know?: Found from here: Environment: Kubernetes version (use ): Cloud provider or hardware configuration: OS (e.g: ): Kernel (e.g. ): Install tools: Network plugin and version (if this is a network-related bug): Others: /assign cc Should we do something for v1.17? (v1.17 code freeze now, but still have the opportunity to fix) can be removed safely in v1.18, right?
/sig scheduling
Working on this in /sig network /priority critical-urgent /assign
Sorry, I didn't realize it. I should have a search before firing an issue. I recommend adding this issue to your PR. e.g. `
Updated, thanks!
/triage unresolved Comment when the issue is assessed and confirmed. I am a bot run by vllry. ‍
This seems to be writing to a cached (should be const) service. I think we need to DeepCopy() it before we mutate the Go type. does that sound right? Is this still our best state of the art?
yes, copy before change
we have a tool to catch this (the cache mutation detector), but it got disabled in our test runs two years ago (xref )
yes great catch - I was always convinced this is enabled...

Discussion:
// controllerName is a unique value used with LabelManagedBy to indicated // the component managing an EndpointSlice. controllerName = "endpointslice-controller.k8s.io" <del> // managedBySetupAnnotation is set on a Service to indicate that // EndpointSlices for the Service have already been configured with // LabelManagedBy. If this annotation is not set, all related EndpointSlices // will have LabelManagedBy set to reference this controller if the label // is not already set. Once all EndpointSlices are labeled, the Controller // will set this annotation on the Service. managedBySetupAnnotation = "endpointslice.kubernetes.io/managed-by-setup" // managedBySetupCompleteValue represents the value of the // managedBySetupAnnotation that indicates that the setup process has been // completed for a Service. managedBySetupCompleteValue = "true" </del> ) // NewController creates and initializes a new Controller

return err } <del> // With the goal of different controllers being able to manage different // subsets of EndpointSlices, LabelManagedBy has been added to indicate // which controller or entity manages an EndpointSlice. As part of this // v1.16->v1.17 change, EndpointSlices will initially be assumed to be // managed by this controller unless a label is set to indicate otherwise. // To ensure a seamless upgrade process, the managedBySetupAnnotation is // used to indicate that LabelManagedBy has been set initially for related // EndpointSlices. If it hasn't been set to the expected value here, we call // ensureSetupManagedByAnnotation() to set up LabelManagedBy on each // EndpointSlice. // TODO(robscott): Remove this before v1.18. err = c.ensureSetupManagedByAnnotation(service) if err != nil { c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToSetEndpointSliceManagedByLabel", "Error adding managed-by Label to Endpoint Slices for Service %s/%s: %v", service.Namespace, service.Name, err) return err } </del> esLabelSelector := labels.Set(map[string]string{ discovery.LabelServiceName: service.Name, discovery.LabelManagedBy:   controllerName,

c.queue.Add(key) } <del> // ensureSetupManagedByAnnotation selects all EndpointSlices for a Service and // ensures they have LabelManagedBy set appropriately. This ensures that only // one controller or entity is trying to manage a given EndpointSlice. This // function provides backwards compatibility with the initial alpha release of // EndpointSlices that did not include these labels. // TODO(robscott): Remove this in time for v1.18. func (c *Controller) ensureSetupManagedByAnnotation(service *v1.Service) error { if managedBySetup, ok := service.Annotations[managedBySetupAnnotation]; ok && managedBySetup == managedBySetupCompleteValue { return nil } esLabelSelector := labels.Set(map[string]string{discovery.LabelServiceName: service.Name}).AsSelectorPreValidated() endpointSlices, err := c.endpointSliceLister.EndpointSlices(service.Namespace).List(esLabelSelector) if err != nil { c.eventRecorder.Eventf(service, v1.EventTypeWarning, "FailedToListEndpointSlices", "Error listing Endpoint Slices for Service %s/%s: %v", service.Namespace, service.Name, err) return err } for _, endpointSlice := range endpointSlices { if _, ok := endpointSlice.Labels[discovery.LabelManagedBy]; !ok { if endpointSlice.Labels == nil { endpointSlice.Labels = make(map[string]string) } endpointSlice.Labels[discovery.LabelManagedBy] = controllerName _, err = c.client.DiscoveryV1beta1().EndpointSlices(endpointSlice.Namespace).Update(endpointSlice) if err != nil { return err } } } if service.Annotations == nil { service.Annotations = make(map[string]string) } service.Annotations[managedBySetupAnnotation] = managedBySetupCompleteValue _, err = c.client.CoreV1().Services(service.Namespace).Update(service) return err } </del> func (c *Controller) addPod(obj interface{}) { pod := obj.(*v1.Pod) services, err := c.serviceSelectorCache.GetPodServiceMemberships(c.serviceLister, pod)

import ( "fmt" <del> "reflect" </del> "testing" "time"

numActionsBefore := len(client.Actions()) + 1 standardSyncService(t, esController, ns, serviceName, "false") <del> if len(client.Actions()) != numActionsBefore+5 { t.Errorf("Expected 5 more actions, got %d", len(client.Actions())-numActionsBefore) </del> <ins> if len(client.Actions()) != numActionsBefore+2 { t.Errorf("Expected 2 more actions, got %d", len(client.Actions())-numActionsBefore) </ins> } <del> // endpointslice should have LabelsManagedBy set as part of update. </del> <ins> // only 2 slices should match, 2 should be deleted, 1 should be updated as a placeholder </ins> expectAction(t, client.Actions(), numActionsBefore, "update", "endpointslices") <del> // service should have managedBySetupAnnotation set as part of update. expectAction(t, client.Actions(), numActionsBefore+1, "update", "services") // only 3 slices should match, 2 of those should be deleted, 1 should be updated as a placeholder expectAction(t, client.Actions(), numActionsBefore+2, "update", "endpointslices") expectAction(t, client.Actions(), numActionsBefore+3, "delete", "endpointslices") expectAction(t, client.Actions(), numActionsBefore+4, "delete", "endpointslices") </del> <ins> expectAction(t, client.Actions(), numActionsBefore+1, "delete", "endpointslices") </ins> } // Ensure SyncService handles a variety of protocols and IPs appropriately.

}}, slice.Endpoints) } <del> func TestEnsureSetupManagedByAnnotation(t *testing.T) { serviceName := "testing-1" testCases := map[string]struct { serviceAnnotation   string startingSliceLabels map[string]string expectedSliceLabels map[string]string }{ "already-labeled": { serviceAnnotation: "foo", startingSliceLabels: map[string]string{ discovery.LabelServiceName: serviceName, discovery.LabelManagedBy:   controllerName, }, expectedSliceLabels: map[string]string{ discovery.LabelServiceName: serviceName, discovery.LabelManagedBy:   controllerName, }, }, "already-annotated": { serviceAnnotation: managedBySetupCompleteValue, startingSliceLabels: map[string]string{ discovery.LabelServiceName: serviceName, discovery.LabelManagedBy:   "other-controller", }, expectedSliceLabels: map[string]string{ discovery.LabelServiceName: serviceName, discovery.LabelManagedBy:   "other-controller", }, }, "missing-and-extra-label": { serviceAnnotation: "foo", startingSliceLabels: map[string]string{ discovery.LabelServiceName: serviceName, "foo":                      "bar", }, expectedSliceLabels: map[string]string{ discovery.LabelServiceName: serviceName, discovery.LabelManagedBy:   controllerName, "foo":                      "bar", }, }, "different-service": { serviceAnnotation: "foo", startingSliceLabels: map[string]string{ discovery.LabelServiceName: "something-else", }, expectedSliceLabels: map[string]string{ discovery.LabelServiceName: "something-else", }, }, } for name, testCase := range testCases { t.Run(name, func(t *testing.T) { client, esController := newController([]string{"node-1"}) ns := metav1.NamespaceDefault service := createService(t, esController, ns, serviceName, testCase.serviceAnnotation) endpointSlice := &discovery.EndpointSlice{ ObjectMeta: metav1.ObjectMeta{ Name:      "testing", Namespace: ns, Labels:    testCase.startingSliceLabels, }, } err := esController.endpointSliceStore.Add(endpointSlice) if err != nil { t.Fatalf("Expected no error adding EndpointSlice: %v", err) } _, err = client.DiscoveryV1beta1().EndpointSlices(ns).Create(endpointSlice) if err != nil { t.Fatalf("Expected no error creating EndpointSlice: %v", err) } esController.ensureSetupManagedByAnnotation(service) updatedService, err := client.CoreV1().Services(ns).Get(service.Name, metav1.GetOptions{}) if err != nil { t.Fatalf("Expected no error getting Service: %v", err) } if updatedService.Annotations[managedBySetupAnnotation] != managedBySetupCompleteValue { t.Errorf("Expected managedBySetupAnnotation: %+v, got: %+v", managedBySetupCompleteValue, updatedService.Annotations[managedBySetupAnnotation]) } updatedSlice, err := client.DiscoveryV1beta1().EndpointSlices(ns).Get(endpointSlice.Name, metav1.GetOptions{}) if err != nil { t.Fatalf("Expected no error getting EndpointSlice: %v", err) } if !reflect.DeepEqual(updatedSlice.Labels, testCase.expectedSliceLabels) { t.Errorf("Expected labels: %+v, got: %+v", updatedSlice.Labels, testCase.expectedSliceLabels) } }) } } </del> // Test helpers func standardSyncService(t *testing.T, esController *endpointSliceController, namespace, serviceName, managedBySetup string) {

Name:              serviceName, Namespace:         namespace, CreationTimestamp: metav1.NewTime(time.Now()), <del> Annotations:       map[string]string{managedBySetupAnnotation: managedBySetup}, </del> }, Spec: v1.ServiceSpec{ Ports:    []v1.ServicePort{{TargetPort: intstr.FromInt(80)}},

kubectl run testmetadata --image=nginx --replicas=2 --port=80 --expose --service-overrides='{ "metadata": { "annotations": { "zone-context": "home" } } } ' # Check result kube::test::get_object_assert deployment "{{range.items}}{{$id_field}}:{{end}}" 'testmetadata:' <del> kube::test::get_object_assert 'service testmetadata' "{{.metadata.annotations}}" "map[endpointslice.kubernetes.io/managed-by-setup:true zone-context:home]" </del> <ins> kube::test::get_object_assert 'service testmetadata' "{{.metadata.annotations}}" "map[zone-context:home]" </ins> ### Expose deployment as a new service # Command kubectl expose deployment testmetadata  --port=1000 --target-port=80 --type=NodePort --name=exposemetadata --overrides='{ "metadata": { "annotations": { "zone-context": "work" } } } ' # Check result <del> kube::test::get_object_assert 'service exposemetadata' "{{.metadata.annotations}}" "map[endpointslice.kubernetes.io/managed-by-setup:true zone-context:work]" </del> <ins> kube::test::get_object_assert 'service exposemetadata' "{{.metadata.annotations}}" "map[zone-context:work]" </ins> # Clean-Up # Command