Repository: Azure/aad-pod-identity
Pull request #309 (2019-07-25 02:56:36)

Liveness probe
<!-- Thank you for helping AAD Pod Identity with a pull request! -->

**Reason for Change**:
<!-- What does this PR improve or fix in AAD Pod Identity? Why is it needed? -->
- Add liveliness at /healthz endpoint for MIC and NMI at 8080 by default.
~~- Add readiness at /ready endpoint for MIC and NMI at 8080 by default.~~
~~- Both liveness and readiness probe return back 200 okay as soon as they are activated from the current run.~~
- The liveliness probe provides "Active" or "Not Active" status as per the state of the component. In case of MIC if the given instance is the elected leader then "Active" string is presented in the /healthz endpoint. For NMI it would show "Active" as soon as the iptables are put in place. 
- Fix the NMI iptables to run instantly instead of waiting for the first ticker instance.

**Issue Fixed**:
<!-- If this PR fixes GitHub issue 1234, add "Fixes #1234" to the next line. -->
#288 

**Notes for Reviewers**:

kkmsft: Ran 13 of 13 Specs in 2388.736 seconds
SUCCESS! -- 13 Passed | 0 Failed | 0 Pending | 0 Skipped
--- PASS: TestAADPodIdentity (2388.74s)
PASS
ok      github.com/Azure/aad-pod-identity/test/e2e      2388.765s

Patch:
diff --git a/cmd/mic/main.go b/cmd/mic/main.go
index 170f2d48..579710aa 100644
--- a/cmd/mic/main.go
+++ b/cmd/mic/main.go
@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/Azure/aad-pod-identity/pkg/mic"
+	"github.com/Azure/aad-pod-identity/pkg/probes"
 	"github.com/Azure/aad-pod-identity/version"
 	"github.com/golang/glog"
 	"k8s.io/client-go/rest"
@@ -19,6 +20,7 @@ var (
 	versionInfo       bool
 	syncRetryDuration time.Duration
 	leaderElectionCfg mic.LeaderElectionConfig
+	httpProbePort     string
 )
 
 func main() {
@@ -39,6 +41,9 @@ func main() {
 	flag.StringVar(&leaderElectionCfg.Name, "leader-election-name", "aad-pod-identity-mic", "leader election name")
 	flag.DurationVar(&leaderElectionCfg.Duration, "leader-election-duration", time.Second*15, "leader election duration")
 
+	//Probe port
+	flag.StringVar(&httpProbePort, "http-probe-port", "8080", "http liveliness probe port")
+
 	flag.Parse()
 	if versionInfo {
 		version.PrintVersionAndExit()
@@ -63,6 +68,12 @@ func main() {
 	if err != nil {
 		glog.Fatalf("Could not get the MIC client: %+v", err)
 	}
+
+	// Health probe will always report success once its started.
+	// MIC instance will report the contents as "Active" only once its elected the leader
+	// and starts the sync loop.
+	probes.InitAndStart(httpProbePort, &micClient.SyncLoopStarted)
+
 	// Starts the leader election loop
 	micClient.Run()
 	glog.Info("AAD Pod identity controller initialized!!")
diff --git a/cmd/nmi/main.go b/cmd/nmi/main.go
index d45c5339..497f49ef 100644
--- a/cmd/nmi/main.go
+++ b/cmd/nmi/main.go
@@ -5,6 +5,7 @@ import (
 
 	"github.com/Azure/aad-pod-identity/pkg/k8s"
 	server "github.com/Azure/aad-pod-identity/pkg/nmi/server"
+	"github.com/Azure/aad-pod-identity/pkg/probes"
 	"github.com/Azure/aad-pod-identity/version"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/pflag"
@@ -28,6 +29,7 @@ var (
 	ipTableUpdateTimeIntervalInSeconds = pflag.Int("ipt-update-interval-sec", defaultIPTableUpdateTimeIntervalInSeconds, "update interval of iptables")
 	forceNamespaced                    = pflag.Bool("forceNamespaced", false, "Forces mic to namespace identities, binding, and assignment")
 	micNamespace                       = pflag.String("MICNamespace", "default", "MIC namespace to short circuit MIC token requests")
+	httpProbePort                      = pflag.String("http-probe-port", "8080", "Http health and liveness probe port")
 )
 
 func main() {
@@ -53,6 +55,10 @@ func main() {
 	s.NodeName = *nodename
 	s.IPTableUpdateTimeIntervalInSeconds = *ipTableUpdateTimeIntervalInSeconds
 
+	// Health probe will always report success once its started. The contents
+	// will report "Active" once the iptables rules are set
+	probes.InitAndStart(*httpProbePort, &s.Initialized)
+
 	if err := s.Run(); err != nil {
 		log.Fatalf("%s", err)
 	}
diff --git a/deploy/infra/master/replicaset/deployment-rbac.yaml b/deploy/infra/master/replicaset/deployment-rbac.yaml
index 6455586a..619b0926 100644
--- a/deploy/infra/master/replicaset/deployment-rbac.yaml
+++ b/deploy/infra/master/replicaset/deployment-rbac.yaml
@@ -125,6 +125,12 @@ spec:
         volumeMounts:
         - mountPath: /run/xtables.lock
           name: iptableslock
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 10
+          periodSeconds: 5
       nodeSelector:
         beta.kubernetes.io/os: linux
 ---
@@ -200,6 +206,12 @@ spec:
         - name: k8s-azure-file
           mountPath: /etc/kubernetes/azure.json
           readOnly: true
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 10
+          periodSeconds: 5
       volumes:
       - name: k8s-azure-file
         hostPath:
diff --git a/deploy/infra/master/replicaset/deployment.yaml b/deploy/infra/master/replicaset/deployment.yaml
index 3b4b62c5..8a9b2a59 100644
--- a/deploy/infra/master/replicaset/deployment.yaml
+++ b/deploy/infra/master/replicaset/deployment.yaml
@@ -81,6 +81,12 @@ spec:
         volumeMounts:
         - mountPath: /run/xtables.lock
           name: iptableslock
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 10
+          periodSeconds: 5
       nodeSelector:
         beta.kubernetes.io/os: linux
 ---
@@ -116,6 +122,12 @@ spec:
           - name: k8s-azure-file
             mountPath: /etc/kubernetes/azure.json
             readOnly: true
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 10
+          periodSeconds: 5
       volumes:
       - name: kubeconfig
         hostPath:
diff --git a/pkg/mic/mic.go b/pkg/mic/mic.go
index 05b8489f..8cf1fa27 100644
--- a/pkg/mic/mic.go
+++ b/pkg/mic/mic.go
@@ -55,6 +55,7 @@ type Client struct {
 	EventChannel      chan aadpodid.EventType
 	NodeClient        NodeGetter
 	IsNamespaced      bool
+	SyncLoopStarted   bool
 	syncRetryInterval time.Duration
 
 	syncing       int32 // protect against conucrrent sync's
@@ -125,7 +126,7 @@ func NewMICClient(cloudconfig string, config *rest.Config, isNamespaced bool, sy
 
 // Run - Initiates the leader election run call to find if its leader and run it
 func (c *Client) Run() {
-	glog.Infof("MIC Leader election initiated")
+	glog.Infof("Initiating MIC Leader election")
 	c.leaderElector.Run()
 }
 
@@ -217,6 +218,7 @@ func (c *Client) Sync(exit <-chan struct{}) {
 	defer ticker.Stop()
 
 	glog.Info("Sync thread started.")
+	c.SyncLoopStarted = true
 	var event aadpodid.EventType
 	for {
 		select {
diff --git a/pkg/nmi/server/server.go b/pkg/nmi/server/server.go
index 92bb841e..2326fc98 100644
--- a/pkg/nmi/server/server.go
+++ b/pkg/nmi/server/server.go
@@ -46,6 +46,7 @@ type Server struct {
 	IPTableUpdateTimeIntervalInSeconds int
 	IsNamespaced                       bool
 	MICNamespace                       string
+	Initialized                        bool
 }
 
 // NMIResponse is the response returned to caller
@@ -80,6 +81,17 @@ func (s *Server) Run() error {
 	return nil
 }
 
+func (s *Server) updateIPTableRulesInternal() {
+	log.Infof("node(%s) hostip(%s) metadataaddress(%s:%s) nmiport(%s)", s.NodeName, s.HostIP, s.MetadataIP, s.MetadataPort, s.NMIPort)
+
+	if err := iptables.AddCustomChain(s.MetadataIP, s.MetadataPort, s.HostIP, s.NMIPort); err != nil {
+		log.Fatalf("%s", err)
+	}
+	if err := iptables.LogCustomChain(); err != nil {
+		log.Fatalf("%s", err)
+	}
+}
+
 // updateIPTableRules ensures the correct iptable rules are set
 // such that metadata requests are received by nmi assigned port
 // NOT originating from HostIP destined to metadata endpoint are
@@ -91,6 +103,11 @@ func (s *Server) updateIPTableRules() {
 	ticker := time.NewTicker(time.Second * time.Duration(s.IPTableUpdateTimeIntervalInSeconds))
 	defer ticker.Stop()
 
+	// Run once before the waiting on ticker for the rules to take effect
+	// immediately.
+	s.updateIPTableRulesInternal()
+	s.Initialized = true
+
 loop:
 	for {
 		select {
@@ -99,13 +116,7 @@ loop:
 			break loop
 
 		case <-ticker.C:
-			log.Infof("node(%s) hostip(%s) metadataaddress(%s:%s) nmiport(%s)", s.NodeName, s.HostIP, s.MetadataIP, s.MetadataPort, s.NMIPort)
-			if err := iptables.AddCustomChain(s.MetadataIP, s.MetadataPort, s.HostIP, s.NMIPort); err != nil {
-				log.Fatalf("%s", err)
-			}
-			if err := iptables.LogCustomChain(); err != nil {
-				log.Fatalf("%s", err)
-			}
+			s.updateIPTableRulesInternal()
 		}
 	}
 }
diff --git a/pkg/probes/probes.go b/pkg/probes/probes.go
new file mode 100644
index 00000000..1814bdaa
--- /dev/null
+++ b/pkg/probes/probes.go
@@ -0,0 +1,45 @@
+package probes
+
+import (
+	"net/http"
+
+	"github.com/golang/glog"
+)
+
+// InitHealthProbe - sets up a health probe which responds with success (200 - OK) once its initialized.
+// The contents of the healthz endpoint will be the string "Active" if the condition is satisfied.
+// The condition is set to true when the sync cycle has become active in case of MIC and the iptables
+// rules set in case of NMI.
+func InitHealthProbe(condition *bool) {
+	http.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(200)
+		if *condition {
+			w.Write([]byte("Active"))
+		} else {
+			w.Write([]byte("Not Active"))
+		}
+	})
+}
+
+func startAsync(port string) {
+	err := http.ListenAndServe(":"+port, nil)
+	if err != nil {
+		glog.Fatalf("Http listen and serve error: %+v", err)
+	} else {
+		glog.V(1).Infof("Http listen and serve started !")
+	}
+}
+
+//Start - Starts the required http server to start the probe to respond.
+func Start(port string) {
+	go startAsync(port)
+}
+
+// InitAndStart - Initialize the default probes and starts the http listening port.
+func InitAndStart(port string, condition *bool) {
+	InitHealthProbe(condition)
+	glog.V(1).Infof("Initialized health probe")
+	// start the probe.
+	Start(port)
+	glog.Infof("Initialized and started health probe")
+}


Test patch:
diff --git a/test/common/k8s/infra/infra.go b/test/common/k8s/infra/infra.go
index de0feb4f..ded19719 100644
--- a/test/common/k8s/infra/infra.go
+++ b/test/common/k8s/infra/infra.go
@@ -11,8 +11,14 @@ import (
 )
 
 // CreateInfra will deploy all the infrastructure components (nmi and mic) on a Kubernetes cluster
-func CreateInfra(namespace, registry, nmiVersion, micVersion, templateOutputPath string) error {
-	t, err := template.New("deployment-rbac.yaml").ParseFiles(path.Join("template", "deployment-rbac.yaml"))
+func CreateInfra(namespace, registry, nmiVersion, micVersion, templateOutputPath string, old bool) error {
+	var err error
+	var t *template.Template
+	if !old {
+		t, err = template.New("deployment-rbac.yaml").ParseFiles(path.Join("template", "deployment-rbac.yaml"))
+	} else {
+		t, err = template.New("deployment-rbac-old.yaml").ParseFiles(path.Join("template", "deployment-rbac-old.yaml"))
+	}
 	if err != nil {
 		return errors.Wrap(err, "Failed to parse deployment-rbac.yaml")
 	}
@@ -20,7 +26,7 @@ func CreateInfra(namespace, registry, nmiVersion, micVersion, templateOutputPath
 	deployFilePath := path.Join(templateOutputPath, namespace+"-deployment.yaml")
 	deployFile, err := os.Create(deployFilePath)
 	if err != nil {
-		return errors.Wrap(err, "Failed to create a deployment file from deployment-rbac.yaml")
+		return errors.Wrap(err, "Failed to create a deployment file")
 	}
 	defer deployFile.Close()
 
diff --git a/test/e2e/aadpodidentity_test.go b/test/e2e/aadpodidentity_test.go
index fc81878a..de683bd6 100644
--- a/test/e2e/aadpodidentity_test.go
+++ b/test/e2e/aadpodidentity_test.go
@@ -2,6 +2,7 @@ package aadpodidentity
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"math/rand"
 	"os"
@@ -24,6 +25,8 @@ import (
 	"github.com/Azure/aad-pod-identity/test/common/util"
 	"github.com/Azure/aad-pod-identity/test/e2e/config"
 	"github.com/pkg/errors"
+	corev1 "k8s.io/api/core/v1"
+	rl "k8s.io/client-go/tools/leaderelection/resourcelock"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -39,6 +42,8 @@ const (
 var (
 	cfg                config.Config
 	templateOutputPath = path.Join("template", "_output")
+	testIndex          = 0
+	noCleanup          = false
 )
 
 var _ = BeforeSuite(func() {
@@ -52,12 +57,15 @@ var _ = BeforeSuite(func() {
 	c, err := config.ParseConfig()
 	Expect(err).NotTo(HaveOccurred())
 	cfg = *c
-
 	setupInfra(cfg.Registry, cfg.NMIVersion, cfg.MICVersion)
 })
 
 var _ = AfterSuite(func() {
 	fmt.Println("\nTearing down the test suite environment...")
+	if noCleanup {
+		fmt.Println("Test failed. No cleanup performed")
+		return
+	}
 
 	// Uninstall CRDs and delete MIC and NMI
 	err := deploy.Delete("default", templateOutputPath)
@@ -102,9 +110,18 @@ var _ = AfterSuite(func() {
 })
 
 var _ = Describe("Kubernetes cluster using aad-pod-identity", func() {
+	BeforeEach(func() {
+		testIndex++
+		fmt.Printf("\n\n[%d] %s\n", testIndex, CurrentGinkgoTestDescription().TestText)
+	})
+
 	AfterEach(func() {
 		fmt.Println("\nTearing down the test environment...")
-
+		if CurrentGinkgoTestDescription().Failed {
+			noCleanup = true
+			fmt.Println("Test failed. No cleanup performed")
+			return
+		}
 		// Ensure a clean cluster after the end of each test
 		cmd := exec.Command("kubectl", "delete", "AzureIdentity,AzureIdentityBinding", "--all")
 		util.PrintCommand(cmd)
@@ -573,7 +590,7 @@ var _ = Describe("Kubernetes cluster using aad-pod-identity", func() {
 		Expect(ok).To(Equal(true))
 
 		// setup mic and nmi with old releases
-		setupInfra("mcr.microsoft.com/k8s/aad-pod-identity", "1.4", "1.3")
+		setupInfraOld("mcr.microsoft.com/k8s/aad-pod-identity", "1.4", "1.3")
 
 		setUpIdentityAndDeployment(keyvaultIdentity, "", "1")
 
@@ -667,15 +684,68 @@ var _ = Describe("Kubernetes cluster using aad-pod-identity", func() {
 		Expect(err).NotTo(HaveOccurred())
 		Expect(exists).To(Equal(true))
 	})
+
+	It("liveness probe test", func() {
+		pods, err := pod.GetAllNameByPrefix("mic")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pods).NotTo(BeNil())
+
+		leader, err := getMICLeader()
+		Expect(err).NotTo(HaveOccurred())
+		fmt.Printf("MIC leader: %s\n", leader)
+
+		for _, p := range pods {
+			// Leader MIC will show as active and other as Not Active
+			if strings.EqualFold(p, leader) {
+				checkHealthProbe(p, "Active")
+			} else {
+				checkHealthProbe(p, "Not Active")
+			}
+		}
+
+		pods, err = pod.GetAllNameByPrefix("nmi")
+		Expect(err).NotTo(HaveOccurred())
+		Expect(pods).NotTo(BeNil())
+		for _, p := range pods {
+			checkHealthProbe(p, "Active")
+		}
+	})
 })
 
-// setupInfra creates the crds, mic, nmi and blocks until iptable entries exist
-func setupInfra(registry, nmiVersion, micVersion string) {
-	// Install CRDs and deploy MIC and NMI
-	err := infra.CreateInfra("default", registry, nmiVersion, micVersion, templateOutputPath)
+func getMICLeader() (string, error) {
+	cmd := exec.Command("kubectl", "get", "endpoints", "aad-pod-identity-mic", "-o", "json")
+	output, err := cmd.CombinedOutput()
+	Expect(err).NotTo(HaveOccurred())
+
+	ep := &corev1.Endpoints{}
+	if err := json.Unmarshal(output, &ep); err != nil {
+		return "", errors.Wrap(err, "Failed to unmarshall json")
+	}
+
+	leRecordStr := ep.Annotations["control-plane.alpha.kubernetes.io/leader"]
+	if leRecordStr == "" {
+		return "", fmt.Errorf("Leader election record empty ")
+	}
 
+	leRecord := &rl.LeaderElectionRecord{}
+	if err := json.Unmarshal([]byte(leRecordStr), leRecord); err != nil {
+		return "", errors.Wrapf(err, "Could not unmarshall: %s. Error: %+v", leRecordStr, err)
+	}
+	return leRecord.HolderIdentity, nil
+}
+
+func checkProbe(p string, endpoint string) string {
+	output, err := pod.RunCommandInPod("exec", p, "--", "wget", "http://127.0.0.1:8080/"+endpoint, "-q", "-O", "-")
 	Expect(err).NotTo(HaveOccurred())
+	fmt.Printf("Output: %s\n", output)
+	return output
+}
+
+func checkHealthProbe(p string, state string) {
+	Expect(strings.EqualFold(state, checkProbe(p, "healthz"))).To(BeTrue())
+}
 
+func checkInfra() {
 	ok, err := daemonset.WaitOnReady(nmiDaemonSet)
 	Expect(err).NotTo(HaveOccurred())
 	Expect(ok).To(Equal(true))
@@ -708,6 +778,22 @@ func setupInfra(registry, nmiVersion, micVersion string) {
 	Expect(ok).To(Equal(true))
 }
 
+// setupInfra creates the crds, mic, nmi and blocks until iptable entries exist
+func setupInfraOld(registry, nmiVersion, micVersion string) {
+	// Install CRDs and deploy MIC and NMI
+	err := infra.CreateInfra("default", registry, nmiVersion, micVersion, templateOutputPath, true)
+	Expect(err).NotTo(HaveOccurred())
+	checkInfra()
+}
+
+// setupInfra creates the crds, mic, nmi and blocks until iptable entries exist
+func setupInfra(registry, nmiVersion, micVersion string) {
+	// Install CRDs and deploy MIC and NMI
+	err := infra.CreateInfra("default", registry, nmiVersion, micVersion, templateOutputPath, false)
+	Expect(err).NotTo(HaveOccurred())
+	checkInfra()
+}
+
 // setUpIdentityAndDeployment will deploy AzureIdentity, AzureIdentityBinding, and an identity validator
 // Suffix will give the tests the option to add a suffix to the end of the identity name, useful for scale tests
 // replicas to indicate the number of replicas for the deployment
diff --git a/test/e2e/template/deployment-rbac-old.yaml b/test/e2e/template/deployment-rbac-old.yaml
new file mode 100644
index 00000000..0bf959f5
--- /dev/null
+++ b/test/e2e/template/deployment-rbac-old.yaml
@@ -0,0 +1,195 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aad-pod-id-nmi-service-account
+  namespace: {{.Namespace}}
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: azureassignedidentities.aadpodidentity.k8s.io
+spec:
+  group: aadpodidentity.k8s.io
+  version: v1
+  names:
+    kind: AzureAssignedIdentity
+    plural: azureassignedidentities
+  scope: Namespaced
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: azureidentitybindings.aadpodidentity.k8s.io
+spec:
+  group: aadpodidentity.k8s.io
+  version: v1
+  names:
+    kind: AzureIdentityBinding
+    plural: azureidentitybindings
+  scope: Namespaced
+---
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: azureidentities.aadpodidentity.k8s.io
+spec:
+  group: aadpodidentity.k8s.io
+  version: v1
+  names:
+    kind: AzureIdentity
+    singular: azureidentity
+    plural: azureidentities
+  scope: Namespaced
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aad-pod-id-nmi-role
+rules:
+- apiGroups: ["apiextensions.k8s.io"]
+  resources: ["customresourcedefinitions"]
+  verbs: ["get", "list"]
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list"]
+- apiGroups: ["aadpodidentity.k8s.io"]
+  resources: ["azureidentitybindings", "azureidentities"]
+  verbs: ["get", "list"]
+- apiGroups: ["aadpodidentity.k8s.io"]
+  resources: ["azureassignedidentities"]
+  verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: aad-pod-id-nmi-binding
+  labels:
+    k8s-app: aad-pod-id-nmi-binding
+subjects:
+- kind: ServiceAccount
+  name: aad-pod-id-nmi-service-account
+  namespace: {{.Namespace}}
+roleRef:
+  kind: ClusterRole
+  name: aad-pod-id-nmi-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: extensions/v1beta1
+kind: DaemonSet
+metadata:
+  labels:
+    kubernetes.io/cluster-service: "true"
+    component: nmi
+    tier: node
+    k8s-app: aad-pod-id
+  name: nmi
+  namespace: {{.Namespace}}
+spec:
+  updateStrategy:
+    type: RollingUpdate
+  template:
+    metadata:
+      labels:
+        component: nmi
+        tier: node
+    spec:
+      serviceAccountName: aad-pod-id-nmi-service-account
+      hostNetwork: true
+      containers:
+      - name: nmi
+        image: "{{.Registry}}/nmi:{{.NMIVersion}}"
+        imagePullPolicy: Always
+        args:
+          {{if .NMIArg}}- nmi {{ end }}
+          - "--host-ip=$(HOST_IP)"
+          - "--node=$(NODE_NAME)"
+        env:
+          - name: HOST_IP
+            valueFrom:
+              fieldRef:
+                fieldPath: status.podIP
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+        securityContext:
+          privileged: true
+          capabilities:
+            add:
+            - NET_ADMIN
+      nodeSelector:
+        beta.kubernetes.io/os: linux
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: aad-pod-id-mic-service-account
+  namespace: {{.Namespace}}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: aad-pod-id-mic-role
+rules:
+- apiGroups: ["apiextensions.k8s.io"]
+  resources: ["customresourcedefinitions"]
+  verbs: ["*"]
+- apiGroups: [""]
+  resources: ["pods", "nodes"]
+  verbs: [ "list", "watch" ]
+- apiGroups: [""]
+  resources: ["events"]
+  verbs: ["create", "patch"]
+- apiGroups: ["aadpodidentity.k8s.io"]
+  resources: ["azureidentitybindings", "azureidentities"]
+  verbs: ["get", "list", "watch", "post"]
+- apiGroups: ["aadpodidentity.k8s.io"]
+  resources: ["azureassignedidentities"]
+  verbs: ["*"]
+---
+apiVersion: rbac.authorization.k8s.io/v1beta1
+kind: ClusterRoleBinding
+metadata:
+  name: aad-pod-id-mic-binding
+  labels:
+    k8s-app: aad-pod-id-mic-binding
+subjects:
+- kind: ServiceAccount
+  name: aad-pod-id-mic-service-account
+  namespace: {{.Namespace}}
+roleRef:
+  kind: ClusterRole
+  name: aad-pod-id-mic-role
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: extensions/v1beta1
+kind: Deployment
+metadata:
+  labels:
+    component: mic
+    k8s-app: aad-pod-id
+  name: mic
+  namespace: {{.Namespace}}
+spec:
+  template:
+    metadata:
+      labels:
+        component: mic
+    spec:
+      serviceAccountName: aad-pod-id-mic-service-account
+      containers:
+      - name: mic
+        image: "{{.Registry}}/mic:{{.MICVersion}}"
+        imagePullPolicy: Always
+        args:
+          {{if .MICArg}}- mic {{ end }}
+          - "--cloudconfig=/etc/kubernetes/azure.json"
+          - "--logtostderr"
+        volumeMounts:
+          - name: k8s-azure-file
+            mountPath: /etc/kubernetes/azure.json
+            readOnly: true
+      volumes:
+      - name: k8s-azure-file
+        hostPath:
+          path: /etc/kubernetes/azure.json
diff --git a/test/e2e/template/deployment-rbac.yaml b/test/e2e/template/deployment-rbac.yaml
index 5260b03a..dc6bcf4e 100644
--- a/test/e2e/template/deployment-rbac.yaml
+++ b/test/e2e/template/deployment-rbac.yaml
@@ -117,6 +117,12 @@ spec:
           capabilities:
             add:
             - NET_ADMIN
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 10
+          periodSeconds: 5
       nodeSelector:
         beta.kubernetes.io/os: linux
 ---
@@ -193,7 +199,13 @@ spec:
           - name: k8s-azure-file
             mountPath: /etc/kubernetes/azure.json
             readOnly: true
+        livenessProbe:
+          httpGet:
+            path: /healthz
+            port: 8080
+          initialDelaySeconds: 10
+          periodSeconds: 5
       volumes:
       - name: k8s-azure-file
         hostPath:
-          path: /etc/kubernetes/azure.json
\ No newline at end of file
+          path: /etc/kubernetes/azure.json