Repository: Azure/aad-pod-identity
Pull request #265 (2019-06-25 08:59:33)

Adds support for MIC to authenticate with azure using system assigned/user assigned MSI
This PR adds the capability for MIC to look at azure.json or environment variables
to determine whether the system assigned or user assigned MSI has to be used for accessing
azure resources. The MIC requests for token based on MSI. Also contains changes in NMI to determine if the request is originating from an MIC replicaset. If so, NMI directly generates the tokens instead of looking up the azure assigned identity for the pod-binding match.

<!-- Thank you for helping AAD Pod Identity with a pull request! -->

**Reason for Change**:
<!-- What does this PR improve or fix in AAD Pod Identity? Why is it needed? -->
Adds ability for MIC to authenticate using system assigned/user assigned MSI.

**Issue Fixed**:
<!-- If this PR fixes GitHub issue 1234, add "Fixes #1234" to the next line. -->

Resolves the item in #261.

**Notes for Reviewers**:
TODO: run the e2e on system assigned identity cluster.


Patch:
diff --git a/cmd/nmi/main.go b/cmd/nmi/main.go
index 7e4f30d1..d45c5339 100644
--- a/cmd/nmi/main.go
+++ b/cmd/nmi/main.go
@@ -27,6 +27,7 @@ var (
 	nodename                           = pflag.String("node", "", "node name")
 	ipTableUpdateTimeIntervalInSeconds = pflag.Int("ipt-update-interval-sec", defaultIPTableUpdateTimeIntervalInSeconds, "update interval of iptables")
 	forceNamespaced                    = pflag.Bool("forceNamespaced", false, "Forces mic to namespace identities, binding, and assignment")
+	micNamespace                       = pflag.String("MICNamespace", "default", "MIC namespace to short circuit MIC token requests")
 )
 
 func main() {
@@ -43,7 +44,7 @@ func main() {
 		log.Fatalf("%+v", err)
 	}
 	*forceNamespaced = *forceNamespaced || "true" == os.Getenv("FORCENAMESPACED")
-	s := server.NewServer(*forceNamespaced)
+	s := server.NewServer(*forceNamespaced, *micNamespace)
 	s.KubeClient = client
 	s.MetadataIP = *metadataIP
 	s.MetadataPort = *metadataPort
diff --git a/docs/readmes/README.msi.md b/docs/readmes/README.msi.md
new file mode 100644
index 00000000..81a8febd
--- /dev/null
+++ b/docs/readmes/README.msi.md
@@ -0,0 +1,50 @@
+## Introduction
+
+The MIC component in aad-pod-identity needs to authenticate with the cloud to assign and remove user assigned identities onto
+virtual machines (VMAs) or virtual machine scale sets(VMSS). This authentication is performed using either the cluster credentials
+obtained from azure.json in AKS/aks-engine clusters or credentials given via environment variables.
+
+MIC can authenticate using the following options:
+1. Service principal
+2. System assigned MSI
+3. User assigned MSI
+
+The rest of the README describes the prerequisite role assignments to be performed for using MSI and how to configure MIC to use system assigned/user assigned MSI.
+
+## Pre-requisites - role assignments
+MIC is responsible for performing operations such as assigning user assigned identity to the underlying vm or vmss which makes up the
+nodes in the Kubernetes cluster. The system/user assigned MSI needs to have role assignments authorizing such operations on the vms/vmss
+and also operations on the user assigned identity.
+
+After the cluster is created, run these commands to retrieve the principal id:
+for VMAS:
+
+```bash
+az vm identity show -g <resource group> -n <vm name> -o yaml
+```
+for VMSS:
+```bash
+az vmss identity show -g <resource group>  -n <vmss scalset name> -o yaml
+```
+
+The type in the output of the above command will identify the system assigned or user assigned MSI. Please record the corresponding
+principal id.
+
+For creating a role assignment to authorize assignment/removal of user assigned identities on VMS/VMSS, run the following command:
+```bash
+az role assignment create --role "Contributor" --assignee <principal id from az vm/vmss identity command>  --scope /subscriptions/<sub id>/resourcegroups/<resource group name>
+```
+
+Now to ensure that the operations are allowed on individual identity, perform the following for every identity in use:
+```bash
+az role assignment create --role "Managed Identity Operator" --assignee <principal id from az vm/vmss identity command>  --scope /subscriptions/<subscription id>/resourcegroups/<resource group name>/providers/Microsoft.ManagedIdentity/userAssignedIdentities/<identity name>
+```
+
+
+## Authentication method
+In case the azure.json is used, the following keys indicates whether the cluster is configured with system assigned or user assigned identity:
+```UseManagedIdentityExtension``` shows that MSI is to be used. If ```UserAssignedIdentityID``` is set, then the user assigned
+identity is used, otherwise system assigned identity is used for authentication.
+
+In case where the azure.json is not used and the environment variables are used, the following variables are used to setup the configuration:
+```USE_MSI``` is to setup MSI. If the ```USER_ASSIGNED_MSI_CLIENT_ID``` is used then user assigned identity is used, otherwise system assigned identity is used.
\ No newline at end of file
diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go
index bb7739e4..3ab9b778 100644
--- a/pkg/auth/auth.go
+++ b/pkg/auth/auth.go
@@ -11,6 +11,28 @@ const (
 	activeDirectoryEndpoint = "https://login.microsoftonline.com/"
 )
 
+// GetServicePrincipalTokenFromMSI return the token for the assigned user
+func GetServicePrincipalTokenFromMSI(resource string) (*adal.Token, error) {
+	// Get the MSI endpoint accoriding with the OS (Linux/Windows)
+	msiEndpoint, err := adal.GetMSIVMEndpoint()
+	if err != nil {
+		return nil, fmt.Errorf("Failed to get the MSI endpoint. Error: %v", err)
+	}
+	// Set up the configuration of the service principal
+	spt, err := adal.NewServicePrincipalTokenFromMSI(msiEndpoint, resource)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to acquire a token for MSI. Error: %v", err)
+	}
+	// Effectively acquire the token
+	err = spt.Refresh()
+	if err != nil {
+		return nil, err
+	}
+	token := spt.Token()
+
+	return &token, nil
+}
+
 // GetServicePrincipalTokenFromMSIWithUserAssignedID return the token for the assigned user
 func GetServicePrincipalTokenFromMSIWithUserAssignedID(clientID, resource string) (*adal.Token, error) {
 	// Get the MSI endpoint accoriding with the OS (Linux/Windows)
@@ -31,7 +53,7 @@ func GetServicePrincipalTokenFromMSIWithUserAssignedID(clientID, resource string
 		return nil, err
 	}
 
-	// Evectively acqurie the token
+	// Effectively acquire the token
 	err = spt.Refresh()
 	if err != nil {
 		return nil, err
diff --git a/pkg/cloudprovider/cloudprovider.go b/pkg/cloudprovider/cloudprovider.go
index e64288b9..c3197ca7 100644
--- a/pkg/cloudprovider/cloudprovider.go
+++ b/pkg/cloudprovider/cloudprovider.go
@@ -7,6 +7,7 @@ import (
 	"os"
 	"path"
 	"regexp"
+	"strings"
 	"time"
 
 	config "github.com/Azure/aad-pod-identity/pkg/config"
@@ -59,6 +60,8 @@ func NewCloudProvider(configFile string) (c *Client, e error) {
 		azureConfig.SubscriptionID = os.Getenv("SUBSCRIPTION_ID")
 		azureConfig.ResourceGroupName = os.Getenv("RESOURCE_GROUP")
 		azureConfig.VMType = os.Getenv("VM_TYPE")
+		azureConfig.UseManagedIdentityExtension = strings.EqualFold(os.Getenv("USE_MSI"), "True")
+		azureConfig.UserAssignedIdentityID = os.Getenv("USER_ASSIGNED_MSI_CLIENT_ID")
 	}
 
 	azureEnv, err := azure.EnvironmentFromName(azureConfig.Cloud)
@@ -77,15 +80,42 @@ func NewCloudProvider(configFile string) (c *Client, e error) {
 		glog.Errorf("Create OAuth config error: %+v", err)
 		return nil, err
 	}
-	spt, err := adal.NewServicePrincipalToken(
-		*oauthConfig,
-		azureConfig.ClientID,
-		azureConfig.ClientSecret,
-		azureEnv.ResourceManagerEndpoint,
-	)
-	if err != nil {
-		glog.Errorf("Get service principle token error: %+v", err)
-		return nil, err
+
+	var spt *adal.ServicePrincipalToken
+	if azureConfig.UseManagedIdentityExtension {
+		// MSI endpoint is required for both types of MSI - system assigned and user assigned.
+		msiEndpoint, err := adal.GetMSIVMEndpoint()
+		if err != nil {
+			glog.Errorf("Failed to get MSI endpoint. Error: %+v", err)
+			return nil, err
+		}
+		// UserAssignedIdentityID is empty, so we are going to use system assigned MSI
+		if azureConfig.UserAssignedIdentityID == "" {
+			glog.Infof("MIC using system assigned identity for authentication.")
+			spt, err = adal.NewServicePrincipalTokenFromMSI(msiEndpoint, azureEnv.ResourceManagerEndpoint)
+			if err != nil {
+				glog.Errorf("Get token from system assigned MSI error: %+v", err)
+				return nil, err
+			}
+		} else { // User assigned identity usage.
+			glog.Infof("MIC using user assigned identity: %s for authentication.", azureConfig.UserAssignedIdentityID)
+			spt, err = adal.NewServicePrincipalTokenFromMSIWithUserAssignedID(msiEndpoint, azureEnv.ResourceManagerEndpoint, azureConfig.UserAssignedIdentityID)
+			if err != nil {
+				glog.Errorf("Get token from user assigned MSI error: %+v", err)
+				return nil, err
+			}
+		}
+	} else { // This is the default scenario - use service principal to get the token.
+		spt, err = adal.NewServicePrincipalToken(
+			*oauthConfig,
+			azureConfig.ClientID,
+			azureConfig.ClientSecret,
+			azureEnv.ResourceManagerEndpoint,
+		)
+		if err != nil {
+			glog.Errorf("Get service principle token error: %+v", err)
+			return nil, err
+		}
 	}
 
 	extClient := compute.NewVirtualMachineExtensionsClient(azureConfig.SubscriptionID)
@@ -166,6 +196,7 @@ func (c *Client) UpdateUserMSI(addUserAssignedMSIIDs []string, removeUserAssigne
 		requiresUpdate = requiresUpdate || addedToList
 	}
 	if requiresUpdate {
+		glog.Infof("Updating user assigned MSIs on %s", node.Name)
 		timeStarted := time.Now()
 		if err := updateFunc(); err != nil {
 			return err
diff --git a/pkg/cloudprovider/vmss.go b/pkg/cloudprovider/vmss.go
index d243e2b8..36c24125 100644
--- a/pkg/cloudprovider/vmss.go
+++ b/pkg/cloudprovider/vmss.go
@@ -108,7 +108,9 @@ func (i *vmssIdentityInfo) RemoveUserIdentity(id string) error {
 	if err := filterUserIdentity(&i.info.Type, i.info.IdentityIds, id); err != nil {
 		return err
 	}
-	if i.info.Type == compute.ResourceIdentityTypeNone {
+	// If we have either no identity assigned or have the system assigned identity only, then we need to set the
+	// IdentityIds list as nil.
+	if i.info.Type == compute.ResourceIdentityTypeNone || i.info.Type == compute.ResourceIdentityTypeSystemAssigned {
 		i.info.IdentityIds = nil
 	}
 	return nil
diff --git a/pkg/config/azureconfig.go b/pkg/config/azureconfig.go
index daf96ac9..60c03e53 100644
--- a/pkg/config/azureconfig.go
+++ b/pkg/config/azureconfig.go
@@ -2,12 +2,14 @@ package config
 
 // AzureConfig is representing /etc/kubernetes/azure.json
 type AzureConfig struct {
-	Cloud             string `json:"cloud" yaml:"cloud"`
-	TenantID          string `json:"tenantId" yaml:"tenantId"`
-	ClientID          string `json:"aadClientId" yaml:"aadClientId"`
-	ClientSecret      string `json:"aadClientSecret" yaml:"aadClientSecret"`
-	SubscriptionID    string `json:"subscriptionId" yaml:"subscriptionId"`
-	ResourceGroupName string `json:"resourceGroup" yaml:"resourceGroup"`
-	SecurityGroupName string `json:"securityGroupName" yaml:"securityGroupName"`
-	VMType            string `json:"vmType" yaml:"vmType"`
+	Cloud                       string `json:"cloud" yaml:"cloud"`
+	TenantID                    string `json:"tenantId" yaml:"tenantId"`
+	ClientID                    string `json:"aadClientId" yaml:"aadClientId"`
+	ClientSecret                string `json:"aadClientSecret" yaml:"aadClientSecret"`
+	SubscriptionID              string `json:"subscriptionId" yaml:"subscriptionId"`
+	ResourceGroupName           string `json:"resourceGroup" yaml:"resourceGroup"`
+	SecurityGroupName           string `json:"securityGroupName" yaml:"securityGroupName"`
+	VMType                      string `json:"vmType" yaml:"vmType"`
+	UseManagedIdentityExtension bool   `json:"useManagedIdentityExtension,omitempty" yaml:"useManagedIdentityExtension,omitempty"`
+	UserAssignedIdentityID      string `json:"userAssignedIdentityID,omitempty" yaml:"userAssignedIdentityID,omitempty"`
 }
diff --git a/pkg/k8s/client.go b/pkg/k8s/client.go
index f7068034..a74c1290 100644
--- a/pkg/k8s/client.go
+++ b/pkg/k8s/client.go
@@ -36,8 +36,8 @@ func getPodPhaseFilter() string {
 
 // Client api client
 type Client interface {
-	// GetPodName return the matching azure identity or nil
-	GetPodName(podip string) (podns, podname string, err error)
+	// GetPodInfo returns the pod name, namespace & replica set name for a given pod ip
+	GetPodInfo(podip string) (podns, podname, rsName string, err error)
 	// ListPodIds pod matching azure identity or nil
 	ListPodIds(podns, podname string) (map[string][]aadpodid.AzureIdentity, error)
 	// GetSecret returns secret the secretRef represents
@@ -82,23 +82,32 @@ func NewKubeClient() (Client, error) {
 	return kubeClient, nil
 }
 
-// GetPodName get pod ns,name from apiserver
-func (c *KubeClient) GetPodName(podip string) (podns, poddname string, err error) {
+func (c *KubeClient) getReplicasetName(pod v1.Pod) string {
+	for _, owner := range pod.OwnerReferences {
+		if strings.EqualFold(owner.Kind, "ReplicaSet") {
+			return owner.Name
+		}
+	}
+	return ""
+}
+
+// GetPodInfo get pod ns,name from apiserver
+func (c *KubeClient) GetPodInfo(podip string) (podns, poddname, rsName string, err error) {
 	if podip == "" {
-		return "", "", fmt.Errorf("podip is empty")
+		return "", "", "", fmt.Errorf("podip is empty")
 	}
 
 	podList, err := c.getPodListRetry(podip, getPodListRetries, getPodListSleepTimeMilliseconds)
 
 	if err != nil {
-		return "", "", err
+		return "", "", "", err
 	}
 	numMatching := len(podList.Items)
 	if numMatching == 1 {
-		return podList.Items[0].Namespace, podList.Items[0].Name, nil
+		return podList.Items[0].Namespace, podList.Items[0].Name, c.getReplicasetName(podList.Items[0]), nil
 	}
 
-	return "", "", fmt.Errorf("match failed, ip:%s matching pods:%v", podip, podList)
+	return "", "", "", fmt.Errorf("match failed, ip:%s matching pods:%v", podip, podList)
 }
 
 func (c *KubeClient) getPodList(podip string) (*v1.PodList, error) {
diff --git a/pkg/k8s/fake.go b/pkg/k8s/fake.go
index 221d47a8..802c98bb 100644
--- a/pkg/k8s/fake.go
+++ b/pkg/k8s/fake.go
@@ -17,9 +17,9 @@ func NewFakeClient() (Client, error) {
 	return fakeClient, nil
 }
 
-// GetPodName returns fake pod name
-func (c *FakeClient) GetPodName(podip string) (podns, podname string, err error) {
-	return "ns", "podname", nil
+// GetPodInfo returns fake pod name, namespace and replicaset
+func (c *FakeClient) GetPodInfo(podip string) (podns, podname, rsName string, err error) {
+	return "ns", "podname", "rsName", nil
 }
 
 // ListPodIds for pod
diff --git a/pkg/nmi/server/server.go b/pkg/nmi/server/server.go
index a1705e42..6e364ff6 100644
--- a/pkg/nmi/server/server.go
+++ b/pkg/nmi/server/server.go
@@ -10,6 +10,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"regexp"
 	"runtime"
 	"runtime/debug"
 	"strings"
@@ -43,6 +44,7 @@ type Server struct {
 	NodeName                           string
 	IPTableUpdateTimeIntervalInSeconds int
 	IsNamespaced                       bool
+	MICNamespace                       string
 }
 
 // NMIResponse is the response returned to caller
@@ -52,9 +54,10 @@ type NMIResponse struct {
 }
 
 // NewServer will create a new Server with default values.
-func NewServer(isNamespaced bool) *Server {
+func NewServer(isNamespaced bool, micNamespace string) *Server {
 	return &Server{
 		IsNamespaced: isNamespaced,
+		MICNamespace: micNamespace,
 	}
 }
 
@@ -216,6 +219,38 @@ func (s *Server) hostHandler(logger *log.Entry, w http.ResponseWriter, r *http.R
 	w.Write(response)
 }
 
+func (s *Server) isMIC(podNS, rsName string) bool {
+	micRegEx := regexp.MustCompile(`^mic-*`)
+	if strings.EqualFold(podNS, s.MICNamespace) && micRegEx.MatchString(rsName) {
+		return true
+	}
+	return false
+}
+
+func (s *Server) getMICToken(logger *log.Entry, rqClientID, rqResource string) ([]byte, int, error) {
+	var token *adal.Token
+	var err error
+	// ClientID is empty, so we are going to use System assigned MSI
+	if rqClientID == "" {
+		logger.Infof("Fetching MIC token for system assigned MSI")
+		token, err = auth.GetServicePrincipalTokenFromMSI(rqResource)
+	} else { // User assigned identity usage.
+		logger.Infof("Fetching MIC token for user assigned MSI for id: %s", rqResource)
+		token, err = auth.GetServicePrincipalTokenFromMSIWithUserAssignedID(rqClientID, rqResource)
+	}
+	if err != nil {
+		logger.Errorf("Failed to get service principal token. Error: %+v", err)
+		// TODO: return the right status code based on the error we got from adal.
+		return nil, http.StatusForbidden, err
+	}
+	response, err := json.Marshal(*token)
+	if err != nil {
+		logger.Errorf("Failed to marshal service principal token. Error: %+v", err)
+		return nil, http.StatusInternalServerError, err
+	}
+	return response, http.StatusOK, nil
+}
+
 // msiHandler uses the remote address to identify the pod ip and uses it
 // to find a matching client id, and then returns the token sourced through
 // AAD using adal
@@ -231,12 +266,26 @@ func (s *Server) msiHandler(logger *log.Entry, w http.ResponseWriter, r *http.Re
 		http.Error(w, msg, http.StatusInternalServerError)
 		return
 	}
-	podns, podname, err := s.KubeClient.GetPodName(podIP)
+	podns, podname, rsName, err := s.KubeClient.GetPodInfo(podIP)
 	if err != nil {
 		logger.Errorf("missing podname for podip:%s, %+v", podIP, err)
 		http.Error(w, err.Error(), http.StatusInternalServerError)
 		return
 	}
+
+	// If its mic, then just directly get the token and pass back.
+	if s.isMIC(podns, rsName) {
+		logger.Infof("MIC pod token handling")
+		response, errorCode, err := s.getMICToken(logger, rqClientID, rqResource)
+		if err != nil {
+			logger.Errorf("failed to get service principal token for pod:%s/%s.  Error code: %d. Error: %+v", podns, podname, errorCode, err)
+			http.Error(w, err.Error(), errorCode)
+			return
+		}
+		w.Write(response)
+		return
+	}
+
 	podIDs, err := listPodIDsWithRetry(r.Context(), s.KubeClient, logger, podns, podname, rqClientID, listPodIDsRetryAttempts)
 	if err != nil {
 		msg := fmt.Sprintf("no AzureAssignedIdentity found for pod:%s/%s in assigned state", podns, podname)
@@ -244,6 +293,7 @@ func (s *Server) msiHandler(logger *log.Entry, w http.ResponseWriter, r *http.Re
 		http.Error(w, msg, http.StatusNotFound)
 		return
 	}
+
 	token, _, err := getTokenForMatchingID(s.KubeClient, logger, rqClientID, rqResource, podIDs)
 	if err != nil {
 		logger.Errorf("failed to get service principal token for pod:%s/%s, %+v", podns, podname, err)


Test patch:
diff --git a/pkg/k8s/client_test.go b/pkg/k8s/client_test.go
index af51d94a..104befc0 100644
--- a/pkg/k8s/client_test.go
+++ b/pkg/k8s/client_test.go
@@ -1,6 +1,7 @@
 package k8s
 
 import (
+	"fmt"
 	"testing"
 
 	v1 "k8s.io/api/core/v1"
@@ -103,3 +104,24 @@ func TestPodListRetries(t *testing.T) {
 	}
 }
 */
+
+func TestGetReplicaSet(t *testing.T) {
+	pod := &v1.Pod{}
+	rsIndex := 1
+	for i := 0; i < 3; i++ {
+		owner := metav1.OwnerReference{}
+		owner.Name = "test" + fmt.Sprintf("%d", i)
+		if i == rsIndex {
+			owner.Kind = "ReplicaSet"
+		} else {
+			owner.Kind = "Kind" + fmt.Sprintf("%d", i)
+		}
+		pod.OwnerReferences = append(pod.OwnerReferences, owner)
+	}
+
+	c := &KubeClient{}
+	rsName := c.getReplicasetName(*pod)
+	if rsName != "test1" {
+		t.Fatalf("Expected rsName: test1. Got: %s", rsName)
+	}
+}