Change:
return newGidTable, nil } <del> //createEndpointService create an endpoint and service in provided namespace. func (plugin *glusterfsPlugin) createEndpointService(namespace string, epServiceName string, hostips []string, pvcname string) (endpoint *v1.Endpoints, service *v1.Service, err error) { addrlist := make([]v1.EndpointAddress, len(hostips)) for i, v := range hostips { addrlist[i].IP = v } endpoint = &v1.Endpoints{ ObjectMeta: metav1.ObjectMeta{ Namespace: namespace, Name:      epServiceName, Labels: map[string]string{ "gluster.kubernetes.io/provisioned-for-pvc": pvcname, }, }, Subsets: []v1.EndpointSubset{{ Addresses: addrlist, Ports:     []v1.EndpointPort{{Port: 1, Protocol: "TCP"}}, }}, } kubeClient := plugin.host.GetKubeClient() if kubeClient == nil { return nil, nil, fmt.Errorf("glusterfs: failed to get kube client when creating endpoint service") } _, err = kubeClient.Core().Endpoints(namespace).Create(endpoint) if err != nil && errors.IsAlreadyExists(err) { glog.V(1).Infof("glusterfs: endpoint [%s] already exist in namespace [%s]", endpoint, namespace) err = nil } if err != nil { glog.Errorf("glusterfs: failed to create endpoint: %v", err) return nil, nil, fmt.Errorf("error creating endpoint: %v", err) } service = &v1.Service{ ObjectMeta: metav1.ObjectMeta{ Name:      epServiceName, Namespace: namespace, Labels: map[string]string{ "gluster.kubernetes.io/provisioned-for-pvc": pvcname, }, }, Spec: v1.ServiceSpec{ Ports: []v1.ServicePort{ {Protocol: "TCP", Port: 1}}}} _, err = kubeClient.Core().Services(namespace).Create(service) if err != nil && errors.IsAlreadyExists(err) { glog.V(1).Infof("glusterfs: service [%s] already exist in namespace [%s]", service, namespace) err = nil } if err != nil { glog.Errorf("glusterfs: failed to create service: %v", err) return nil, nil, fmt.Errorf("error creating service: %v", err) } return endpoint, service, nil } // deleteEndpointService delete the endpoint and service from the provided namespace. func (plugin *glusterfsPlugin) deleteEndpointService(namespace string, epServiceName string) (err error) { kubeClient := plugin.host.GetKubeClient() if kubeClient == nil { return fmt.Errorf("glusterfs: failed to get kube client when deleting endpoint service") } err = kubeClient.Core().Services(namespace).Delete(epServiceName, nil) if err != nil { glog.Errorf("glusterfs: error deleting service %s/%s: %v", namespace, epServiceName, err) return fmt.Errorf("error deleting service %s/%s: %v", namespace, epServiceName, err) } glog.V(1).Infof("glusterfs: service/endpoint %s/%s deleted successfully", namespace, epServiceName) return nil } </del> func (d *glusterfsVolumeDeleter) getGid() (int, bool, error) { gidStr, ok := d.spec.Annotations[volumehelper.VolumeGidAnnotationKey]

Discussion:
several operations to the glusterfs mount operation that nodes are not permitted to perform: looks up arbitrary secrets based on storage class parameters. this will not succeed, as the node authorizer will only allow nodes access to secrets referenced by pods or persistent volumes used by the node attempts to create a service and endpoint using the node's API client. nodes do not have permission to create arbitrary endpoints and services. For both of those reasons, that recreate operation will fail when attempted from the node. The recreate operation should probably move to one of the storage-related controllers (provisioner or attach/detach) cc
Marking this with 1.7 milestone until we have a better idea of whether this is blocking or not for the 1.7 release.
CC Is the endpoint originally created by the user or by the volume plugin creating/deleting Endpoint objects? In addition to the concerns raised, having a volume create new API objects seems like a bad idea (abstraction layer violation?).
From on Slack: Unless there is justification otherwise, I will roll back the PR on Monday.
cc/
revert button doesn't work, have to do it manually
Revert PR is approved and awaiting merge (). Once that merges this issue can be closed.
Opened an E2E pr to catch this kind of behavior in the future. WIP, still exploring a method to reproduce and catch the behavior.