Change:
import ( "fmt" <del> "net/http" "net/http/httptest" </del> "testing" <ins> "time" </ins> "github.com/GoogleCloudPlatform/kubernetes/pkg/api" <del> "github.com/GoogleCloudPlatform/kubernetes/pkg/api/testapi" </del> "github.com/GoogleCloudPlatform/kubernetes/pkg/client" <del> "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/fake" "github.com/GoogleCloudPlatform/kubernetes/pkg/runtime" "github.com/GoogleCloudPlatform/kubernetes/pkg/util" </del> <ins> fake_cloud "github.com/GoogleCloudPlatform/kubernetes/pkg/cloudprovider/fake" </ins> ) <del> func newMinionList(count int) *api.MinionList { </del> <ins> func newMinion(name string) *api.Minion { return &api.Minion{ObjectMeta: api.ObjectMeta{Name: name}} } type FakeMinionHandler struct { client.Fake client.FakeMinions // Input: Hooks determine if request is valid or not CreateHook func(*FakeMinionHandler, *api.Minion) bool Existing   []*api.Minion // Output CreatedMinions []*api.Minion DeletedMinions []*api.Minion RequestCount   int } func (c *FakeMinionHandler) Minions() client.MinionInterface { return c } func (m *FakeMinionHandler) Create(minion *api.Minion) (*api.Minion, error) { defer func() { m.RequestCount++ }() if m.CreateHook == nil || m.CreateHook(m, minion) { m.CreatedMinions = append(m.CreatedMinions, minion) return minion, nil } else { return nil, fmt.Errorf("Create error.") } } func (m *FakeMinionHandler) List() (*api.MinionList, error) { defer func() { m.RequestCount++ }() </ins> minions := []api.Minion{} <del> for i := 0; i < count; i++ { minions = append(minions, api.Minion{ ObjectMeta: api.ObjectMeta{ Name: fmt.Sprintf("minion%d", i), }, }) </del> <ins> for i := 0; i < len(m.Existing); i++ { if !contains(m.Existing[i], m.DeletedMinions) { minions = append(minions, *m.Existing[i]) } </ins> } <del> return &api.MinionList{ Items: minions, </del> <ins> for i := 0; i < len(m.CreatedMinions); i++ { if !contains(m.Existing[i], m.DeletedMinions) { minions = append(minions, *m.CreatedMinions[i]) } </ins> } <ins> return &api.MinionList{Items: minions}, nil </ins> } <del> type serverResponse struct { statusCode int obj        interface{} </del> <ins> func (m *FakeMinionHandler) Delete(id string) error { m.DeletedMinions = append(m.DeletedMinions, newMinion(id)) m.RequestCount++ return nil </ins> } <del> func makeTestServer(t *testing.T, minionResponse serverResponse) (*httptest.Server, *util.FakeHandler) { fakeMinionHandler := util.FakeHandler{ StatusCode:   minionResponse.statusCode, ResponseBody: runtime.EncodeOrDie(testapi.Codec(), minionResponse.obj.(runtime.Object)), } mux := http.NewServeMux() mux.Handle("/api/"+testapi.Version()+"/minions", &fakeMinionHandler) mux.Handle("/api/"+testapi.Version()+"/minions/", &fakeMinionHandler) mux.HandleFunc("/", func(res http.ResponseWriter, req *http.Request) { t.Errorf("unexpected request: %v", req.RequestURI) res.WriteHeader(http.StatusNotFound) }) return httptest.NewServer(mux), &fakeMinionHandler </del> <ins> func TestSyncStaticCreateMinion(t *testing.T) { fakeMinionHandler := &FakeMinionHandler{ CreateHook: func(fake *FakeMinionHandler, minion *api.Minion) bool { return true }, } minionController := NewMinionController(nil, ".*", []string{"minion0"}, &api.NodeResources{}, fakeMinionHandler) if err := minionController.SyncStatic(time.Millisecond); err != nil { t.Errorf("unexpected error: %v", err) } if fakeMinionHandler.RequestCount != 1 { t.Errorf("Expected 1 call, but got %v.", fakeMinionHandler.RequestCount) } if len(fakeMinionHandler.CreatedMinions) != 1 { t.Errorf("expect only 1 minion created, got %v", len(fakeMinionHandler.CreatedMinions)) } if fakeMinionHandler.CreatedMinions[0].Name != "minion0" { t.Errorf("unexpect minion %v created", fakeMinionHandler.CreatedMinions[0].Name) } } func TestSyncStaticCreateMinionWithError(t *testing.T) { fakeMinionHandler := &FakeMinionHandler{ CreateHook: func(fake *FakeMinionHandler, minion *api.Minion) bool { if fake.RequestCount == 0 { return false } return true }, } minionController := NewMinionController(nil, ".*", []string{"minion0"}, &api.NodeResources{}, fakeMinionHandler) if err := minionController.SyncStatic(time.Millisecond); err != nil { t.Errorf("unexpected error: %v", err) } if fakeMinionHandler.RequestCount != 2 { t.Errorf("Expected 2 call, but got %v.", fakeMinionHandler.RequestCount) } if len(fakeMinionHandler.CreatedMinions) != 1 { t.Errorf("expect only 1 minion created, got %v", len(fakeMinionHandler.CreatedMinions)) } if fakeMinionHandler.CreatedMinions[0].Name != "minion0" { t.Errorf("unexpect minion %v created", fakeMinionHandler.CreatedMinions[0].Name) } </ins> } <del> func TestSyncCreateMinion(t *testing.T) { testServer, minionHandler := makeTestServer(t, serverResponse{http.StatusOK, newMinionList(1)}) defer testServer.Close() client := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Version()}) </del> <ins> func TestSyncCloudCreateMinion(t *testing.T) { fakeMinionHandler := &FakeMinionHandler{ Existing: []*api.Minion{newMinion("minion0")}, } </ins> instances := []string{"minion0", "minion1"} fakeCloud := fake_cloud.FakeCloud{ Machines: instances, } <del> minionController := NewMinionController(&fakeCloud, ".*", nil, nil, client) if err := minionController.Sync(); err != nil { </del> <ins> minionController := NewMinionController(&fakeCloud, ".*", nil, nil, fakeMinionHandler) if err := minionController.SyncCloud(); err != nil { </ins> t.Errorf("unexpected error: %v", err) } <del> data := runtime.EncodeOrDie(testapi.Codec(), &api.Minion{ObjectMeta: api.ObjectMeta{Name: "minion1"}}) minionHandler.ValidateRequest(t, "/api/"+testapi.Version()+"/minions", "POST", &data) </del> <ins> if fakeMinionHandler.RequestCount != 2 { t.Errorf("Expected 2 call, but got %v.", fakeMinionHandler.RequestCount) } if len(fakeMinionHandler.CreatedMinions) != 1 { t.Errorf("expect only 1 minion created, got %v", len(fakeMinionHandler.CreatedMinions)) } if fakeMinionHandler.CreatedMinions[0].Name != "minion1" { t.Errorf("unexpect minion %v created", fakeMinionHandler.CreatedMinions[0].Name) } </ins> } <del> func TestSyncDeleteMinion(t *testing.T) { testServer, minionHandler := makeTestServer(t, serverResponse{http.StatusOK, newMinionList(2)}) defer testServer.Close() client := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Version()}) </del> <ins> func TestSyncCloudDeleteMinion(t *testing.T) { fakeMinionHandler := &FakeMinionHandler{ Existing: []*api.Minion{newMinion("minion0"), newMinion("minion1")}, } </ins> instances := []string{"minion0"} fakeCloud := fake_cloud.FakeCloud{ Machines: instances, } <del> minionController := NewMinionController(&fakeCloud, ".*", nil, nil, client) if err := minionController.Sync(); err != nil { </del> <ins> minionController := NewMinionController(&fakeCloud, ".*", nil, nil, fakeMinionHandler) if err := minionController.SyncCloud(); err != nil { </ins> t.Errorf("unexpected error: %v", err) } <del> minionHandler.ValidateRequest(t, "/api/"+testapi.Version()+"/minions/minion1", "DELETE", nil) </del> <ins> if fakeMinionHandler.RequestCount != 2 { t.Errorf("Expected 2 call, but got %v.", fakeMinionHandler.RequestCount) } if len(fakeMinionHandler.DeletedMinions) != 1 { t.Errorf("expect only 1 minion deleted, got %v", len(fakeMinionHandler.DeletedMinions)) } if fakeMinionHandler.DeletedMinions[0].Name != "minion1" { t.Errorf("unexpect minion %v created", fakeMinionHandler.DeletedMinions[0].Name) } </ins> } <del> func TestSyncMinionRegexp(t *testing.T) { testServer, minionHandler := makeTestServer(t, serverResponse{http.StatusOK, newMinionList(1)}) defer testServer.Close() client := client.NewOrDie(&client.Config{Host: testServer.URL, Version: testapi.Version()}) </del> <ins> func TestSyncCloudRegexp(t *testing.T) { fakeMinionHandler := &FakeMinionHandler{ Existing: []*api.Minion{newMinion("minion0")}, } </ins> instances := []string{"minion0", "minion1", "node0"} fakeCloud := fake_cloud.FakeCloud{ Machines: instances, } <del> minionController := NewMinionController(&fakeCloud, "minion[0-9]+", nil, nil, client) if err := minionController.Sync(); err != nil { </del> <ins> minionController := NewMinionController(&fakeCloud, "minion[0-9]+", nil, nil, fakeMinionHandler) if err := minionController.SyncCloud(); err != nil { </ins> t.Errorf("unexpected error: %v", err) } <del> // Only minion1 is created. data := runtime.EncodeOrDie(testapi.Codec(), &api.Minion{ObjectMeta: api.ObjectMeta{Name: "minion1"}}) minionHandler.ValidateRequest(t, "/api/"+testapi.Version()+"/minions", "POST", &data) </del> <ins> if fakeMinionHandler.RequestCount != 2 { t.Errorf("Expected 2 call, but got %v.", fakeMinionHandler.RequestCount) } if len(fakeMinionHandler.CreatedMinions) != 1 { t.Errorf("expect only 1 minion created, got %v", len(fakeMinionHandler.CreatedMinions)) } if fakeMinionHandler.CreatedMinions[0].Name != "minion1" { t.Errorf("unexpect minion %v created", fakeMinionHandler.CreatedMinions[0].Name) } } func contains(minion *api.Minion, minions []*api.Minion) bool { for i := 0; i < len(minions); i++ { if minion.Name == minions[i].Name { return true } } return false </ins> }

Discussion:
gives but gives If minions were already in etcd from some previous launch of the apiserver/controller things work just fine no matter what the order. So to reproduce you have to clear everything from etcd before you try...
can you take a look at this? Probably just need to retry in background until etcd comes up?
Yep, there is no control loop for static machine config. Adding one should be trivial.
Blocked by Basically, apiserver returns inconsistent response.
I have observed a similar sort of thing, but what it appeared to be is more like the is not listening on it's socket at the time tries to connect to it and it never retries. So my work-around is to run before it like so: What I then observed is that the connection is indeed being refused at first attempt, but works on the second.
what do you see in the logs? I was seeing that it basically fails to connect to once and doesn't retry (although it said that it will try to retry).
should fix it, sorry for the trouble wondering where did you see that "it will try to retry"?
that wasn't the exact wording there... let me see if I can find out what exactly it said.
Ok, so what I'm seeing now is: I hope that PIDs would be a good enough proof to what order services have started up: I could provide some systemd graph dumps, but that would probably be too verbose, PIDs seem clearer. So really don't see how got //apisevercontroller-managerapiserver` didn't open the socket yet...