Repository: Azure/aad-pod-identity
Pull request #367 (2019-08-29 17:26:53)

Clean up assigned identities if node not found
<!-- Thank you for helping AAD Pod Identity with a pull request! -->

**Reason for Change**:
<!-- What does this PR improve or fix in AAD Pod Identity? Why is it needed? -->
Delete assigned identities for the node, when node no longer exists.

**Issue Fixed**:
<!-- If this PR fixes GitHub issue 1234, add "Fixes #1234" to the next line. -->
Fixes https://github.com/Azure/aad-pod-identity/issues/366

**Notes for Reviewers**:
Pending testing + running e2e


Patch:
diff --git a/cmd/nmi/main.go b/cmd/nmi/main.go
index 4b4708f8..3799217b 100644
--- a/cmd/nmi/main.go
+++ b/cmd/nmi/main.go
@@ -47,13 +47,14 @@ func main() {
 		log.SetLevel(log.DebugLevel)
 	}
 	log.Infof("Starting nmi process. Version: %v. Build date: %v", version.NMIVersion, version.BuildDate)
-	client, err := k8s.NewKubeClient()
+	logger := &server.Log{}
+
+	client, err := k8s.NewKubeClient(logger)
 	if err != nil {
 		log.Fatalf("%+v", err)
 	}
 	exit := make(<-chan struct{})
-	logger := &server.Log{}
-	client.Start(exit, logger)
+	client.Start(exit)
 	*forceNamespaced = *forceNamespaced || "true" == os.Getenv("FORCENAMESPACED")
 	s := server.NewServer(*forceNamespaced, *micNamespace)
 	s.KubeClient = client
diff --git a/pkg/crd/crd.go b/pkg/crd/crd.go
index afb06d67..f4309527 100644
--- a/pkg/crd/crd.go
+++ b/pkg/crd/crd.go
@@ -9,9 +9,9 @@ import (
 	"github.com/golang/glog"
 
 	aadpodid "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1"
+	inlog "github.com/Azure/aad-pod-identity/pkg/logger"
 	"github.com/Azure/aad-pod-identity/pkg/stats"
 
-	log "github.com/Azure/aad-pod-identity/pkg/logger"
 	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -29,12 +29,13 @@ type Client struct {
 	IDInformer                   cache.SharedInformer
 	AssignedIDInformer           cache.SharedInformer
 	PodIdentityExceptionInformer cache.SharedInformer
+	log                          inlog.Logger
 }
 
 // ClientInt ...
 type ClientInt interface {
 	Start(exit <-chan struct{})
-	SyncCache(exit <-chan struct{})
+	SyncCache(exit <-chan struct{}, initial bool)
 	RemoveAssignedIdentity(assignedIdentity *aadpodid.AzureAssignedIdentity) error
 	CreateAssignedIdentity(assignedIdentity *aadpodid.AzureAssignedIdentity) error
 	UpdateAzureAssignedIdentityStatus(assignedIdentity *aadpodid.AzureAssignedIdentity, status string) error
@@ -46,23 +47,23 @@ type ClientInt interface {
 }
 
 // NewCRDClientLite ...
-func NewCRDClientLite(config *rest.Config) (crdClient *Client, err error) {
+func NewCRDClientLite(config *rest.Config, log inlog.Logger) (crdClient *Client, err error) {
 	restClient, err := newRestClient(config)
 	if err != nil {
-		glog.Error(err)
+		log.Error(err)
 		return nil, err
 	}
 
 	assignedIDListWatch := newAssignedIDListWatch(restClient)
 	assignedIDListInformer, err := newAssignedIDInformer(assignedIDListWatch)
 	if err != nil {
-		glog.Error(err)
+		log.Error(err)
 		return nil, err
 	}
 	podIdentityExceptionListWatch := newPodIdentityExceptionListWatch(restClient)
 	podIdentityExceptionInformer, err := newPodIdentityExceptionInformer(podIdentityExceptionListWatch)
 	if err != nil {
-		glog.Error(err)
+		log.Error(err)
 		return nil, err
 	}
 
@@ -74,31 +75,31 @@ func NewCRDClientLite(config *rest.Config) (crdClient *Client, err error) {
 }
 
 // NewCRDClient returns a new crd client and error if any
-func NewCRDClient(config *rest.Config, eventCh chan aadpodid.EventType) (crdClient *Client, err error) {
+func NewCRDClient(config *rest.Config, eventCh chan aadpodid.EventType, log inlog.Logger) (crdClient *Client, err error) {
 	restClient, err := newRestClient(config)
 	if err != nil {
-		glog.Error(err)
+		log.Error(err)
 		return nil, err
 	}
 
 	bindingListWatch := newBindingListWatch(restClient)
 	bindingInformer, err := newBindingInformer(restClient, eventCh, bindingListWatch)
 	if err != nil {
-		glog.Error(err)
+		log.Error(err)
 		return nil, err
 	}
 
 	idListWatch := newIDListWatch(restClient)
 	idInformer, err := newIDInformer(restClient, eventCh, idListWatch)
 	if err != nil {
-		glog.Error(err)
+		log.Error(err)
 		return nil, err
 	}
 
 	assignedIDListWatch := newAssignedIDListWatch(restClient)
 	assignedIDListInformer, err := newAssignedIDInformer(assignedIDListWatch)
 	if err != nil {
-		glog.Error(err)
+		log.Error(err)
 		return nil, err
 	}
 
@@ -107,6 +108,7 @@ func NewCRDClient(config *rest.Config, eventCh chan aadpodid.EventType) (crdClie
 		BindingInformer:    bindingInformer,
 		IDInformer:         idInformer,
 		AssignedIDInformer: assignedIDListInformer,
+		log:                log,
 	}, nil
 }
 
@@ -132,7 +134,6 @@ func newRestClient(config *rest.Config) (r *rest.RESTClient, err error) {
 	//Client interacting with our CRDs
 	restClient, err := rest.RESTClientFor(&crdconfig)
 	if err != nil {
-		glog.Error(err)
 		return nil, err
 	}
 	return restClient, nil
@@ -232,11 +233,11 @@ func newPodIdentityExceptionInformer(lw *cache.ListWatch) (cache.SharedInformer,
 }
 
 // StartLite to be used only case of lite client
-func (c *Client) StartLite(exit <-chan struct{}, log log.Logger) {
+func (c *Client) StartLite(exit <-chan struct{}) {
 	go c.AssignedIDInformer.Run(exit)
 	go c.PodIdentityExceptionInformer.Run(exit)
-	c.SyncCache(exit)
-	log.Info("CRD lite informers started ")
+	c.SyncCache(exit, true)
+	c.log.Info("CRD lite informers started ")
 }
 
 // Start ...
@@ -244,13 +245,17 @@ func (c *Client) Start(exit <-chan struct{}) {
 	go c.BindingInformer.Run(exit)
 	go c.IDInformer.Run(exit)
 	go c.AssignedIDInformer.Run(exit)
-	c.SyncCache(exit)
-	glog.Info("CRD informers started")
+	c.SyncCache(exit, true)
+	c.log.Info("CRD informers started")
 }
 
 // SyncCache synchronizes cache
-func (c *Client) SyncCache(exit <-chan struct{}) {
+func (c *Client) SyncCache(exit <-chan struct{}, initial bool) {
 	if !cache.WaitForCacheSync(exit) {
+		if !initial {
+			c.log.Errorf("Cache could not be synchronized")
+			return
+		}
 		panic("Cache could not be synchronized")
 	}
 }
@@ -323,7 +328,7 @@ func (c *Client) ListAssignedIDs() (res *[]aadpodid.AzureAssignedIdentity, err e
 		o, ok := assignedID.(*aadpodid.AzureAssignedIdentity)
 		if !ok {
 			err := fmt.Errorf("could not cast %T to %s", assignedID, aadpodid.AzureAssignedIDResource)
-			glog.Error(err)
+			c.log.Error(err)
 			return nil, err
 		}
 		// Note: List items returned from cache have empty Kind and API version..
@@ -351,7 +356,7 @@ func (c *Client) ListIds() (res *[]aadpodid.AzureIdentity, err error) {
 		o, ok := id.(*aadpodid.AzureIdentity)
 		if !ok {
 			err := fmt.Errorf("could not cast %T to %s", id, aadpodid.AzureIDResource)
-			glog.Error(err)
+			c.log.Error(err)
 			return nil, err
 		}
 		// Note: List items returned from cache have empty Kind and API version..
@@ -379,7 +384,7 @@ func (c *Client) ListPodIdentityExceptions(ns string) (res *[]aadpodid.AzurePodI
 		o, ok := binding.(*aadpodid.AzurePodIdentityException)
 		if !ok {
 			err := fmt.Errorf("could not cast %T to %s", binding, aadpodid.AzureIdentityExceptionResource)
-			glog.Error(err)
+			c.log.Error(err)
 			return nil, err
 		}
 		if o.Namespace == ns {
diff --git a/pkg/k8s/client.go b/pkg/k8s/client.go
index fc2ff214..7428fe0a 100644
--- a/pkg/k8s/client.go
+++ b/pkg/k8s/client.go
@@ -15,8 +15,8 @@ import (
 
 	aadpodid "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1"
 	crd "github.com/Azure/aad-pod-identity/pkg/crd"
-	"github.com/Azure/aad-pod-identity/version"
 	inlog "github.com/Azure/aad-pod-identity/pkg/logger"
+	"github.com/Azure/aad-pod-identity/version"
 	"github.com/golang/glog"
 	log "github.com/sirupsen/logrus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,7 +32,7 @@ const (
 // Client api client
 type Client interface {
 	// Start just starts any informers required.
-	Start(<-chan struct{}, inlog.Logger)
+	Start(<-chan struct{})
 	// GetPodInfo returns the pod name, namespace & replica set name for a given pod ip
 	GetPodInfo(podip string) (podns, podname, rsName string, selectors *metav1.LabelSelector, err error)
 	// ListPodIds pod matching azure identity or nil
@@ -50,10 +50,11 @@ type KubeClient struct {
 	// Crd client used to access our CRD resources.
 	CrdClient   *crd.Client
 	PodInformer informersv1.PodInformer
+	log         inlog.Logger
 }
 
 // NewKubeClient new kubernetes api client
-func NewKubeClient() (Client, error) {
+func NewKubeClient(log inlog.Logger) (Client, error) {
 	config, err := buildConfig()
 	if err != nil {
 		return nil, err
@@ -63,7 +64,7 @@ func NewKubeClient() (Client, error) {
 	if err != nil {
 		return nil, err
 	}
-	crdclient, err := crd.NewCRDClientLite(config)
+	crdclient, err := crd.NewCRDClientLite(config, log)
 	if err != nil {
 		return nil, err
 	}
@@ -71,16 +72,21 @@ func NewKubeClient() (Client, error) {
 	informer := informers.NewSharedInformerFactory(clientset, 30*time.Second)
 	podInformer := informer.Core().V1().Pods()
 
-	kubeClient := &KubeClient{CrdClient: crdclient, ClientSet: clientset, PodInformer: podInformer}
+	kubeClient := &KubeClient{
+		CrdClient:   crdclient,
+		ClientSet:   clientset,
+		PodInformer: podInformer,
+		log:         log,
+	}
 
 	return kubeClient, nil
 }
 
 // Start the corresponding starts
-func (c *KubeClient) Start(exit <-chan struct{}, log inlog.Logger) {
+func (c *KubeClient) Start(exit <-chan struct{}) {
 	go c.PodInformer.Informer().Run(exit)
-	c.CrdClient.StartLite(exit, log)
-	c.CrdClient.SyncCache(exit)
+	c.CrdClient.StartLite(exit)
+	c.CrdClient.SyncCache(exit, true)
 }
 
 func (c *KubeClient) getReplicasetName(pod v1.Pod) string {
diff --git a/pkg/k8s/fake.go b/pkg/k8s/fake.go
index a559c9b7..67cfcb70 100644
--- a/pkg/k8s/fake.go
+++ b/pkg/k8s/fake.go
@@ -4,7 +4,6 @@ import (
 	aadpodid "github.com/Azure/aad-pod-identity/pkg/apis/aadpodidentity/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	inlog "github.com/Azure/aad-pod-identity/pkg/logger"
 )
 
 // FakeClient implements Interface
@@ -24,7 +23,6 @@ func (c *FakeClient) GetPodInfo(podip string) (podns, podname, rsName string, se
 	return "ns", "podname", "rsName", nil, nil
 }
 
-
 // ListPodIds for pod
 func (c *FakeClient) ListPodIds(podns, podname string) (map[string][]aadpodid.AzureIdentity, error) {
 	return nil, nil
@@ -41,6 +39,6 @@ func (c *FakeClient) GetSecret(secretRef *v1.SecretReference) (*v1.Secret, error
 }
 
 // Start - for starting informer clients in the fake Client
-func (c *FakeClient) Start(exit <-chan struct{}, log inlog.Logger ) {
+func (c *FakeClient) Start(exit <-chan struct{}) {
 
 }
diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go
index 67857a5d..5aeb0156 100644
--- a/pkg/logger/logger.go
+++ b/pkg/logger/logger.go
@@ -3,5 +3,7 @@ package logger
 // Logger - this interface is used to abstract the logging libraries used in MIC and NMI.
 type Logger interface {
 	Info(msg string)
+	Error(err error)
+	Infof(format string, args ...interface{})
 	Errorf(format string, args ...interface{})
 }
diff --git a/pkg/mic/mic.go b/pkg/mic/mic.go
index c5eae541..757a0c69 100644
--- a/pkg/mic/mic.go
+++ b/pkg/mic/mic.go
@@ -3,6 +3,7 @@ package mic
 import (
 	"fmt"
 	"os"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -92,7 +93,7 @@ func NewMICClient(cloudconfig string, config *rest.Config, isNamespaced bool, sy
 	glog.V(1).Infof("Cloud provider initialized")
 
 	eventCh := make(chan aadpodid.EventType, 100)
-	crdClient, err := crd.NewCRDClient(config, eventCh)
+	crdClient, err := crd.NewCRDClient(config, eventCh, Log{})
 	if err != nil {
 		return nil, err
 	}
@@ -237,6 +238,13 @@ func (c *Client) Sync(exit <-chan struct{}) {
 		begin := time.Now()
 		workDone := false
 
+		cacheTime := time.Now()
+
+		// There is a delay in data propogation to cache. It's possible that the creates performed in the previous sync cycle
+		// are not propogated before this sync cycle began. In order to avoid redoing the cycle, we sync cache again.
+		c.CRDClient.SyncCache(exit, false)
+		stats.Put(stats.CacheSync, time.Since(cacheTime))
+
 		// List all pods in all namespaces
 		systemTime := time.Now()
 		listPods, err := c.PodClient.GetPods()
@@ -706,7 +714,7 @@ func (c *Client) updateNodeAndDeps(newAssignedIDs []aadpodid.AzureAssignedIdenti
 func (c *Client) updateUserMSI(newAssignedIDs []aadpodid.AzureAssignedIdentity, nodeName string, nodeTrackList trackUserAssignedMSIIds, nodeRefs map[string]bool, wg *sync.WaitGroup) {
 	defer wg.Done()
 	beginAdding := time.Now()
-	glog.V(5).Infof("Processing node %s, add [%d], del [%d]", nodeName, len(nodeTrackList.assignedIDsToCreate), len(nodeTrackList.assignedIDsToDelete))
+	glog.Infof("Processing node %s, add [%d], del [%d]", nodeName, len(nodeTrackList.assignedIDsToCreate), len(nodeTrackList.assignedIDsToDelete))
 
 	for _, createID := range nodeTrackList.assignedIDsToCreate {
 		if createID.Status.Status == "" {
@@ -730,7 +738,16 @@ func (c *Client) updateUserMSI(newAssignedIDs []aadpodid.AzureAssignedIdentity,
 
 	node, err := c.NodeClient.Get(nodeName)
 	if err != nil {
-		glog.Errorf("Lookup of identities on node %s resulted in error %v", nodeName, err)
+		if !strings.Contains(err.Error(), "not found") {
+			glog.Errorf("Unable to get node %s while updating user msis. Error %v", nodeName, err)
+			return
+		}
+
+		glog.Warningf("Unable to get node %s while updating user msis. Error %v", nodeName, err)
+
+		// node is no longer found in the cluster, all the assigned identities that were created in this sync loop
+		// and those that already exist for this node need to be deleted.
+		c.cleanUpAllAssignedIdentitiesOnNode(nodeName, nodeTrackList)
 		return
 	}
 
@@ -853,3 +870,35 @@ func (c *Client) updateUserMSI(newAssignedIDs []aadpodid.AzureAssignedIdentity,
 	stats.Put(stats.TotalCreateOrUpdate, time.Since(beginAdding))
 	c.statsMutex.Unlock()
 }
+
+// cleanUpAllAssignedIdentitiesOnNode deletes all assigned identities associated with a the node
+func (c *Client) cleanUpAllAssignedIdentitiesOnNode(node string, nodeTrackList trackUserAssignedMSIIds) {
+	glog.Infof("deleting all assigned identites for node %s", node)
+	for _, createID := range nodeTrackList.assignedIDsToCreate {
+		binding := createID.Spec.AzureBindingRef
+
+		err := c.removeAssignedIdentity(&createID)
+		if err != nil {
+			c.EventRecorder.Event(binding, corev1.EventTypeWarning, "binding remove error",
+				fmt.Sprintf("Removing assigned identity binding %s node %s for pod %s resulted in error %v", binding.Name, createID.Spec.NodeName, createID.Name, err.Error()))
+			glog.Error(err)
+			continue
+		}
+		c.EventRecorder.Event(binding, corev1.EventTypeNormal, "binding removed",
+			fmt.Sprintf("Binding %s removed from node %s for pod %s", binding.Name, createID.Spec.NodeName, createID.Spec.Pod))
+	}
+
+	for _, deleteID := range nodeTrackList.assignedIDsToDelete {
+		binding := deleteID.Spec.AzureBindingRef
+
+		err := c.removeAssignedIdentity(&deleteID)
+		if err != nil {
+			c.EventRecorder.Event(binding, corev1.EventTypeWarning, "binding remove error",
+				fmt.Sprintf("Removing assigned identity binding %s node %s for pod %s resulted in error %v", binding.Name, deleteID.Spec.NodeName, deleteID.Name, err.Error()))
+			glog.Error(err)
+			continue
+		}
+		c.EventRecorder.Event(binding, corev1.EventTypeNormal, "binding removed",
+			fmt.Sprintf("Binding %s removed from node %s for pod %s", binding.Name, deleteID.Spec.NodeName, deleteID.Spec.Pod))
+	}
+}
diff --git a/pkg/mic/utils.go b/pkg/mic/utils.go
index ec0289db..c58acf46 100644
--- a/pkg/mic/utils.go
+++ b/pkg/mic/utils.go
@@ -11,7 +11,17 @@ func (l Log) Info(msg string) {
 	glog.Info(msg)
 }
 
+// Infof - log the messages to be info messages and formatted.
+func (l Log) Infof(format string, args ...interface{}) {
+	glog.Infof(format, &args)
+}
+
 // Errorf - log the messages to be error messages and formatted.
 func (l Log) Errorf(format string, args ...interface{}) {
 	glog.Errorf(format, &args)
 }
+
+// Error - log the messages to be error
+func (l Log) Error(err error) {
+	glog.Error(err)
+}
diff --git a/pkg/nmi/server/utils.go b/pkg/nmi/server/utils.go
index 2242b0e8..9155851f 100644
--- a/pkg/nmi/server/utils.go
+++ b/pkg/nmi/server/utils.go
@@ -15,6 +15,16 @@ func (l Log) Info(msg string) {
 	log.Info(msg)
 }
 
+// Infof - log the messages to be info messages and formatted.
+func (l Log) Infof(format string, args ...interface{}) {
+	log.Infof(format, &args)
+}
+
+// Error - log the messages to be error
+func (l Log) Error(err error) {
+	log.Error(err)
+}
+
 // Errorf - log the messages to be error messages and formatted.
 func (l Log) Errorf(format string, args ...interface{}) {
 	log.Errorf(format, &args)
diff --git a/pkg/stats/stats.go b/pkg/stats/stats.go
index f8c5d327..4802976b 100644
--- a/pkg/stats/stats.go
+++ b/pkg/stats/stats.go
@@ -15,6 +15,7 @@ type StatsType string
 const (
 	Total                StatsType = "Total"
 	System               StatsType = "System"
+	CacheSync            StatsType = "CacheSync"
 	CurrentState         StatsType = "Gather current state"
 	PodList              StatsType = "Pod listing"
 	BindingList          StatsType = "Binding listing"
@@ -81,6 +82,7 @@ func PrintSync() {
 		Print(BindingList)
 		Print(AssignedIDList)
 		Print(System)
+		Print(CacheSync)
 
 		Print(CloudGet)
 		Print(CloudPut)


Test patch:
diff --git a/pkg/mic/mic_test.go b/pkg/mic/mic_test.go
index 3d07e23d..b33d17e6 100644
--- a/pkg/mic/mic_test.go
+++ b/pkg/mic/mic_test.go
@@ -2,6 +2,7 @@ package mic
 
 import (
 	"errors"
+	"fmt"
 	"reflect"
 	"sync"
 	"testing"
@@ -344,7 +345,7 @@ func NewTestCrdClient(config *rest.Config) *TestCrdClient {
 func (c *TestCrdClient) Start(exit <-chan struct{}) {
 }
 
-func (c *TestCrdClient) SyncCache(exit <-chan struct{}) {
+func (c *TestCrdClient) SyncCache(exit <-chan struct{}, initial bool) {
 
 }
 
@@ -482,6 +483,10 @@ func (c *TestNodeClient) Get(name string) (*corev1.Node, error) {
 	return node, nil
 }
 
+func (c *TestNodeClient) Delete(name string) {
+	delete(c.nodes, name)
+}
+
 func (c *TestNodeClient) Start(<-chan struct{}) {}
 
 func (c *TestNodeClient) AddNode(name string, opts ...func(*corev1.Node)) {
@@ -1175,6 +1180,78 @@ func TestSyncRetryLoop(t *testing.T) {
 	}
 }
 
+func TestSyncNodeNotFound(t *testing.T) {
+	eventCh := make(chan aadpodid.EventType, 100)
+	cloudClient := NewTestCloudClient(config.AzureConfig{})
+	crdClient := NewTestCrdClient(nil)
+	podClient := NewTestPodClient()
+	nodeClient := NewTestNodeClient()
+	var evtRecorder TestEventRecorder
+	evtRecorder.lastEvent = new(LastEvent)
+	evtRecorder.eventChannel = make(chan bool, 100)
+
+	micClient := NewMICTestClient(eventCh, cloudClient, crdClient, podClient, nodeClient, &evtRecorder)
+
+	// Add a pod, identity and binding.
+	crdClient.CreateID("test-id1", aadpodid.UserAssignedMSI, "test-user-msi-resourceid", "test-user-msi-clientid", nil, "", "", "")
+	crdClient.CreateBinding("testbinding1", "test-id1", "test-select1")
+
+	for i := 0; i < 10; i++ {
+		nodeClient.AddNode(fmt.Sprintf("test-node%d", i))
+		podClient.AddPod(fmt.Sprintf("test-pod%d", i), "default", fmt.Sprintf("test-node%d", i), "test-select1")
+		eventCh <- aadpodid.PodCreated
+	}
+
+	defer micClient.testRunSync()(t)
+
+	if !evtRecorder.WaitForEvents(10) {
+		t.Fatalf("Timeout waiting for mic sync cycles")
+	}
+
+	listAssignedIDs, err := crdClient.ListAssignedIDs()
+	if err != nil {
+		glog.Error(err)
+		t.Errorf("list assigned failed")
+	}
+	if !(len(*listAssignedIDs) == 10) {
+		t.Fatalf("expected assigned identities len: %d, got: %d", 10, len(*listAssignedIDs))
+	}
+	for i := range *listAssignedIDs {
+		if !((*listAssignedIDs)[i].Status.Status == aadpodid.AssignedIDAssigned) {
+			t.Fatalf("expected status to be %s, got: %s", aadpodid.AssignedIDAssigned, (*listAssignedIDs)[i].Status.Status)
+		}
+	}
+
+	// delete 5 nodes
+	for i := 5; i < 10; i++ {
+		nodeClient.Delete(fmt.Sprintf("test-node%d", i))
+		podClient.DeletePod(fmt.Sprintf("test-pod%d", i), "default")
+		eventCh <- aadpodid.PodDeleted
+	}
+
+	nodeClient.AddNode("test-nodex")
+	podClient.AddPod("test-podx", "default", "test-node1", "test-select1")
+	eventCh <- aadpodid.PodCreated
+
+	if !evtRecorder.WaitForEvents(6) {
+		t.Fatalf("Timeout waiting for mic sync cycles")
+	}
+
+	listAssignedIDs, err = crdClient.ListAssignedIDs()
+	if err != nil {
+		glog.Error(err)
+		t.Errorf("list assigned failed")
+	}
+	if !(len(*listAssignedIDs) == 6) {
+		t.Fatalf("expected assigned identities len: %d, got: %d", 6, len(*listAssignedIDs))
+	}
+	for i := range *listAssignedIDs {
+		if !((*listAssignedIDs)[i].Status.Status == aadpodid.AssignedIDAssigned) {
+			t.Fatalf("expected status to be %s, got: %s", aadpodid.AssignedIDAssigned, (*listAssignedIDs)[i].Status.Status)
+		}
+	}
+}
+
 func TestSyncExit(t *testing.T) {
 	eventCh := make(chan aadpodid.EventType)
 	cloudClient := NewTestCloudClient(config.AzureConfig{VMType: "vmss"})