Change:
Test failed with and the relevant part of the stack trace is: Logs:
(or anyone), integration tests do not log much and I don't know what's going on in volume controller. Would it be possible to raise integration test log level at least to v=3 (4 preferred)?
what is the status of
can you please take a look?
Increasing integration log level can go into I think. It's probably a good idea to split them into their own job.

Discussion:
} func TestPersistentVolumeRecycler(t *testing.T) { <ins> glog.V(2).Infof("TestPersistentVolumeRecycler started") </ins> _, s := framework.RunAMaster(t) defer s.Close()

defer ctrl.Stop() // This PV will be claimed, released, and recycled. <del> pv := createPV("fake-pv", "/tmp/foo", "10G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, api.PersistentVolumeReclaimRecycle) pvc := createPVC("fake-pvc", "5G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}) </del> <ins> pv := createPV("fake-pv-recycler", "/tmp/foo", "10G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, api.PersistentVolumeReclaimRecycle) pvc := createPVC("fake-pvc-recycler", "5G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}) </ins> _, err := testClient.PersistentVolumes().Create(pv) if err != nil { t.Errorf("Failed to create PersistentVolume: %v", err) } <ins> glog.V(2).Infof("TestPersistentVolumeRecycler pvc created") </ins> _, err = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) if err != nil { t.Errorf("Failed to create PersistentVolumeClaim: %v", err) } <ins> glog.V(2).Infof("TestPersistentVolumeRecycler pvc created") </ins> // wait until the controller pairs the volume and claim waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeBound) <ins> glog.V(2).Infof("TestPersistentVolumeRecycler pv bound") </ins> waitForPersistentVolumeClaimPhase(testClient, pvc.Name, watchPVC, api.ClaimBound) <ins> glog.V(2).Infof("TestPersistentVolumeRecycler pvc bound") </ins> // deleting a claim releases the volume, after which it can be recycled if err := testClient.PersistentVolumeClaims(api.NamespaceDefault).Delete(pvc.Name, nil); err != nil { t.Errorf("error deleting claim %s", pvc.Name) } <ins> glog.V(2).Infof("TestPersistentVolumeRecycler pvc deleted") </ins> waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeReleased) <ins> glog.V(2).Infof("TestPersistentVolumeRecycler pv released") </ins> waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeAvailable) <ins> glog.V(2).Infof("TestPersistentVolumeRecycler pv available") } </ins> <del> // end of Recycler test. // Deleter test begins now. // tests are serial because running masters concurrently that delete keys may cause similar tests to time out </del> <ins> func TestPersistentVolumeDeleter(t *testing.T) { glog.V(2).Infof("TestPersistentVolumeDeleter started") _, s := framework.RunAMaster(t) defer s.Close() </ins> deleteAllEtcdKeys() <ins> testClient, ctrl, watchPV, watchPVC := createClients(t, s) defer watchPV.Stop() defer watchPVC.Stop() ctrl.Run() defer ctrl.Stop() </ins> <del> // change the reclamation policy of the PV for the next test pv.Spec.PersistentVolumeReclaimPolicy = api.PersistentVolumeReclaimDelete </del> <ins> // This PV will be claimed, released, and deleted. pv := createPV("fake-pv-deleter", "/tmp/foo", "10G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, api.PersistentVolumeReclaimDelete) pvc := createPVC("fake-pvc-deleter", "5G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}) </ins> <del> _, err = testClient.PersistentVolumes().Create(pv) </del> <ins> _, err := testClient.PersistentVolumes().Create(pv) </ins> if err != nil { t.Errorf("Failed to create PersistentVolume: %v", err) } <ins> glog.V(2).Infof("TestPersistentVolumeDeleter pv created") </ins> _, err = testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(pvc) if err != nil { t.Errorf("Failed to create PersistentVolumeClaim: %v", err) } <ins> glog.V(2).Infof("TestPersistentVolumeDeleter pvc created") </ins> waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeBound) <ins> glog.V(2).Infof("TestPersistentVolumeDeleter pv bound") </ins> waitForPersistentVolumeClaimPhase(testClient, pvc.Name, watchPVC, api.ClaimBound) <ins> glog.V(2).Infof("TestPersistentVolumeDeleter pvc bound") </ins> // deleting a claim releases the volume, after which it can be recycled if err := testClient.PersistentVolumeClaims(api.NamespaceDefault).Delete(pvc.Name, nil); err != nil { t.Errorf("error deleting claim %s", pvc.Name) } <ins> glog.V(2).Infof("TestPersistentVolumeDeleter pvc deleted") </ins> waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeReleased) <ins> glog.V(2).Infof("TestPersistentVolumeDeleter pv released") </ins> for { event := <-watchPV.ResultChan()

break } } <ins> glog.V(2).Infof("TestPersistentVolumeDeleter pv deleted") } func TestPersistentVolumeBindRace(t *testing.T) { // Test a race binding many claims to a PV that is pre-bound to a specific // PVC. Only this specific PVC should get bound. glog.V(2).Infof("TestPersistentVolumeBindRace started") _, s := framework.RunAMaster(t) defer s.Close() </ins> <del> // test the race between claims and volumes.  ensure only a volume only binds to a single claim. </del> deleteAllEtcdKeys() <ins> testClient, ctrl, watchPV, watchPVC := createClients(t, s) defer watchPV.Stop() defer watchPVC.Stop() ctrl.Run() defer ctrl.Stop() pv := createPV("fake-pv-race", "/tmp/foo", "10G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}, api.PersistentVolumeReclaimRetain) pvc := createPVC("fake-pvc-race", "5G", []api.PersistentVolumeAccessMode{api.ReadWriteOnce}) </ins> counter := 0 maxClaims := 100 claims := []*api.PersistentVolumeClaim{}

counter += 1 clone, _ := conversion.NewCloner().DeepCopy(pvc) newPvc, _ := clone.(*api.PersistentVolumeClaim) <del> newPvc.ObjectMeta = api.ObjectMeta{Name: fmt.Sprintf("fake-pvc-%d", counter)} </del> <ins> newPvc.ObjectMeta = api.ObjectMeta{Name: fmt.Sprintf("fake-pvc-race-%d", counter)} </ins> claim, err := testClient.PersistentVolumeClaims(api.NamespaceDefault).Create(newPvc) if err != nil { t.Fatal("Error creating newPvc: %v", err) } claims = append(claims, claim) } <ins> glog.V(2).Infof("TestPersistentVolumeBindRace claims created") </ins> // putting a bind manually on a pv should only match the claim it is bound to rand.Seed(time.Now().Unix())

t.Fatalf("Unexpected error getting claimRef: %v", err) } pv.Spec.ClaimRef = claimRef <ins> pv.Spec.ClaimRef.UID = "" </ins> pv, err = testClient.PersistentVolumes().Create(pv) if err != nil { t.Fatalf("Unexpected error creating pv: %v", err) } <ins> glog.V(2).Infof("TestPersistentVolumeBindRace pv created, pre-bound to %s", claim.Name) </ins> waitForPersistentVolumePhase(testClient, pv.Name, watchPV, api.VolumeBound) <ins> glog.V(2).Infof("TestPersistentVolumeBindRace pv bound") </ins> waitForAnyPersistentVolumeClaimPhase(watchPVC, api.ClaimBound) <ins> glog.V(2).Infof("TestPersistentVolumeBindRace pvc bound") </ins> pv, err = testClient.PersistentVolumes().Get(pv.Name) if err != nil {