Repository: kubernetes
Issue: Inconsistent behavior using multiple api-servers with different ServiceCIDRs 
Labels: kind/bug,sig/network,sig/api-machinery,triage/accepted

Kubernetes apiservers are responsible of handling multiple aspects of Services:
- apiservers run a reconcile loop to maintain the special `kubernetes.default` Service, that uses the first address of the configured primary Service CIDR
- apiservers assign `ClusterIP` to Services by sharing a bitmap that is snapshoted in `etcd`
- apiservers run a repair loop to keep the Services ClusterIPs and bitmap in sync

The Service CIDRs are configured by the following flag `--service-cluster-ip-range string` , and is it possible that mulitple apiservers use different configuration.

If this happens, the cluster will be in a totally inconsistent state, since each loop in each apiserver will try to reconcile their own parameters, 

```sh
    kube_apiserver_test.go:684: svcs from client 2: svc-apiserver2-9 192.168.0.22
    kube_apiserver_test.go:687: ------------
    kube_apiserver_test.go:658: Unexpected error: Internal error occurred: failed to allocate a serviceIP: the provided range does not match the current range
I1229 20:22:50.079653   56647 alloc.go:327] "allocated clusterIPs" service="default/svc-apiserver2-17" clusterIPs=map[IPv4:192.168.0.78]
    kube_apiserver_test.go:673: ------------
    kube_apiserver_test.go:676: svcs from client 1: kubernetes 10.0.0.1
    kube_apiserver_test.go:676: svcs from client 1: svc-apiserver2-0 192.168.0.114
    kube_apiserver_test.go:676: svcs from client 1: svc-apiserver2-1 192.168.0.248
```

easy to reproduce with the following integration test

```diff
diff --git a/test/integration/controlplane/kube_apiserver_test.go b/test/integration/controlplane/kube_apiserver_test.go
index 0658516aa70..f7b302636aa 100644
--- a/test/integration/controlplane/kube_apiserver_test.go
+++ b/test/integration/controlplane/kube_apiserver_test.go
@@ -42,6 +42,7 @@ import (
        "k8s.io/client-go/kubernetes"
        "k8s.io/kube-aggregator/pkg/apis/apiregistration"
        "k8s.io/kube-openapi/pkg/validation/spec"
+       "k8s.io/kubernetes/cmd/kube-apiserver/app/options"
        kubeapiservertesting "k8s.io/kubernetes/cmd/kube-apiserver/app/testing"
        "k8s.io/kubernetes/test/integration/etcd"
        "k8s.io/kubernetes/test/integration/framework"
@@ -606,3 +607,84 @@ func TestMultiAPIServerNodePortAllocation(t *testing.T) {
        }

 }
+
+func TestMultiAPIServerDifferentServiceCIDRs(t *testing.T) {
+       serviceObject := &corev1.Service{
+               ObjectMeta: metav1.ObjectMeta{
+                       Labels: map[string]string{"foo": "bar"},
+                       Name:   "test-svc",
+               },
+               Spec: corev1.ServiceSpec{
+                       Ports: []corev1.ServicePort{
+                               {
+                                       Name:       "test",
+                                       Port:       443,
+                                       TargetPort: intstr.IntOrString{IntVal: 443},
+                                       Protocol:   "TCP",
+                               },
+                       },
+                       Type:     corev1.ServiceTypeNodePort,
+                       Selector: map[string]string{"foo": "bar"},
+               },
+       }
+       etcd := framework.SharedEtcd()
+
+       serviceCIDR1 := "10.0.0.0/16"
+
+       client1, _, tearDownFn1 := framework.StartTestServer(t, framework.TestServerSetup{
+               ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+                       opts.ServiceClusterIPRanges = serviceCIDR1
+                       opts.Etcd.StorageConfig = *etcd
+               },
+       })
+       defer tearDownFn1()
+
+       serviceCIDR2 := "192.168.0.0/24"
+
+       client2, _, tearDownFn2 := framework.StartTestServer(t, framework.TestServerSetup{
+               ModifyServerRunOptions: func(opts *options.ServerRunOptions) {
+                       opts.ServiceClusterIPRanges = serviceCIDR2
+                       opts.Etcd.StorageConfig = *etcd
+               },
+       })
+       defer tearDownFn2()
+
+       for i := 0; i < 2500; i++ {
+               // create a Service with first API server
+               svc1 := serviceObject.DeepCopy()
+               svc1.Name = fmt.Sprintf("svc-apiserver1-%d", i)
+               _, err := client1.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc1, metav1.CreateOptions{})
+               if err != nil {
+                       t.Logf("Unexpected error: %v", err)
+               }
+
+               // create a Service with second API server
+               svc2 := serviceObject.DeepCopy()
+               svc2.Name = fmt.Sprintf("svc-apiserver2-%d", i)
+               _, err = client2.CoreV1().Services(metav1.NamespaceDefault).Create(context.Background(), svc2, metav1.CreateOptions{})
+               if err != nil {
+                       t.Logf("Unexpected error: %v", err)
+               }
+
+               svcs, err := client1.CoreV1().Services(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
+               if err != nil {
+                       t.Logf("Unexpected error: %v", err)
+               }
+               t.Logf("------------")
+
+               for _, s := range svcs.Items {
+                       t.Logf("svcs from client 1: %s %s\n", s.Name, s.Spec.ClusterIP)
+               }
+
+               svcs, err = client1.CoreV1().Services(metav1.NamespaceAll).List(context.TODO(), metav1.ListOptions{})
+               if err != nil {
+                       t.Logf("Unexpected error: %v", err)
+               }
+               for _, s := range svcs.Items {
+                       t.Logf("svcs from client 2: %s %s\n", s.Name, s.Spec.ClusterIP)
+               }
+               time.Sleep(2 * time.Second)
+               t.Logf("------------")
+       }
+
+}
```

It is important to mention that the Service definition is not reconciled since 1.17 https://github.com/kubernetes/kubernetes/commit/2a9a9fa155e92d0bd7f4eb85d843cfa14b830cca
It means that first apiserver in create the `kubernetes.default` Service wins

Reporting it just for documenting it, initially described and fixed by https://github.com/kubernetes/enhancements/tree/master/keps/sig-network/1880-multiple-service-cidrs

/sig api-machinery
/sig network
/assign