Repository: kubernetes/kops (go)

Issue:
Environment variable to control exported user
It would be great to have an environment variable to be able to configure the user kops exports.

We have RBAC setup with Google OIDC for auditing purposes and when we run commands like `kops replace` or `kops export kubecfg`, it overwrites the current kube config context with the default certificate based admin user. Then any commands we run with kubectl show up as the admin user rather than the individual users.

In this case, it would be great to have kops not add the admin user to the kubecfg file and just use the value from the environment variable in the context block. The environment variable could be something like `KOPS_KUBECFG_USER`


Pull request description:
Specify user on export kubecfg
* Don't export admin user by default.
* Allow specifying existing user when exporting context.

Fixes #8017 


Patch:
diff --git a/cmd/kops/create_cluster.go b/cmd/kops/create_cluster.go
index 6d99233882..0d88f40566 100644
--- a/cmd/kops/create_cluster.go
+++ b/cmd/kops/create_cluster.go
@@ -1360,14 +1360,12 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr
 		updateClusterOptions.Target = c.Target
 		updateClusterOptions.Models = c.Models
 		updateClusterOptions.OutDir = c.OutDir
+		updateClusterOptions.admin = true
+		updateClusterOptions.CreateKubecfg = true
 
 		// SSHPublicKey has already been mapped
 		updateClusterOptions.SSHPublicKey = ""
 
-		// No equivalent options:
-		//  updateClusterOptions.MaxTaskDuration = c.MaxTaskDuration
-		//  updateClusterOptions.CreateKubecfg = c.CreateKubecfg
-
 		_, err := RunUpdateCluster(ctx, f, cluster.Name, out, updateClusterOptions)
 		if err != nil {
 			return err
@@ -1388,7 +1386,7 @@ func RunCreateCluster(ctx context.Context, f *util.Factory, out io.Writer, c *Cr
 				fmt.Fprintf(&sb, " * edit your master instance group: kops edit ig --name=%s %s\n", cluster.Name, masters[0].ObjectMeta.Name)
 			}
 			fmt.Fprintf(&sb, "\n")
-			fmt.Fprintf(&sb, "Finally configure your cluster with: kops update cluster --name %s --yes\n", cluster.Name)
+			fmt.Fprintf(&sb, "Finally configure your cluster with: kops update cluster --name %s --yes --admin\n", cluster.Name)
 			fmt.Fprintf(&sb, "\n")
 
 			_, err := out.Write(sb.Bytes())
diff --git a/cmd/kops/export_kubecfg.go b/cmd/kops/export_kubecfg.go
index f3939d104f..fa3dabce4a 100644
--- a/cmd/kops/export_kubecfg.go
+++ b/cmd/kops/export_kubecfg.go
@@ -34,14 +34,17 @@ import (
 
 var (
 	exportKubecfgLong = templates.LongDesc(i18n.T(`
-	Export a kubecfg file for a cluster from the state store. The configuration
-	will be saved into a users $HOME/.kube/config file.
-	To export the kubectl configuration to a specific file set the KUBECONFIG
-	environment variable.`))
+	Export a kubecfg file for a cluster from the state store. By default the configuration
+	will be saved into a users $HOME/.kube/config file. Kops will respect the KUBECONFIG environment variable
+	if the --kubeconfig flag is not set.
+	`))
 
 	exportKubecfgExample = templates.Examples(i18n.T(`
-	# export a kubecfg file
-	kops export kubecfg kubernetes-cluster.example.com
+	# export a kubeconfig file with the cluster admin user (make sure you keep this user safe!)
+	kops export kubecfg kubernetes-cluster.example.com --admin
+
+	# export using a user already existing in the kubeconfig file
+	kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user
 		`))
 
 	exportKubecfgShort = i18n.T(`Export kubecfg.`)
@@ -50,6 +53,8 @@ var (
 type ExportKubecfgOptions struct {
 	KubeConfigPath string
 	all            bool
+	admin          bool
+	user           string
 }
 
 func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command {
@@ -69,8 +74,10 @@ func NewCmdExportKubecfg(f *util.Factory, out io.Writer) *cobra.Command {
 		},
 	}
 
-	cmd.Flags().StringVar(&options.KubeConfigPath, "kubeconfig", options.KubeConfigPath, "The location of the kubeconfig file to create.")
+	cmd.Flags().StringVar(&options.KubeConfigPath, "kubeconfig", options.KubeConfigPath, "the location of the kubeconfig file to create.")
 	cmd.Flags().BoolVar(&options.all, "all", options.all, "export all clusters from the kops state store")
+	cmd.Flags().BoolVar(&options.admin, "admin", options.admin, "export the cluster admin user and add it to the context")
+	cmd.Flags().StringVar(&options.user, "user", options.user, "add an existing user to the cluster context")
 
 	return cmd
 }
@@ -80,12 +87,17 @@ func RunExportKubecfg(ctx context.Context, f *util.Factory, out io.Writer, optio
 	if err != nil {
 		return err
 	}
-
-	var clusterList []*kopsapi.Cluster
 	if options.all {
 		if len(args) != 0 {
-			return fmt.Errorf("Cannot use both --all flag and positional arguments")
+			return fmt.Errorf("cannot use both --all flag and positional arguments")
 		}
+	}
+	if options.admin && options.user != "" {
+		return fmt.Errorf("cannot use both --admin and --user")
+	}
+
+	var clusterList []*kopsapi.Cluster
+	if options.all {
 		list, err := clientset.ListClusters(ctx, metav1.ListOptions{})
 		if err != nil {
 			return err
@@ -116,7 +128,7 @@ func RunExportKubecfg(ctx context.Context, f *util.Factory, out io.Writer, optio
 			return err
 		}
 
-		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options))
+		conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, buildPathOptions(options), options.admin, options.user)
 		if err != nil {
 			return err
 		}
diff --git a/cmd/kops/update_cluster.go b/cmd/kops/update_cluster.go
index 564ba79784..526f169ef4 100644
--- a/cmd/kops/update_cluster.go
+++ b/cmd/kops/update_cluster.go
@@ -52,7 +52,7 @@ var (
 
 	updateClusterExample = templates.Examples(i18n.T(`
 	# After cluster has been edited or upgraded, configure it with:
-	kops update cluster k8s-cluster.example.com --yes --state=s3://kops-state-1234 --yes
+	kops update cluster k8s-cluster.example.com --yes --state=s3://kops-state-1234 --yes --admin
 	`))
 
 	updateClusterShort = i18n.T("Update a cluster.")
@@ -65,7 +65,10 @@ type UpdateClusterOptions struct {
 	OutDir          string
 	SSHPublicKey    string
 	RunTasksOptions fi.RunTasksOptions
-	CreateKubecfg   bool
+
+	CreateKubecfg bool
+	admin         bool
+	user          string
 
 	Phase string
 
@@ -80,7 +83,7 @@ func (o *UpdateClusterOptions) InitDefaults() {
 	o.Models = strings.Join(cloudup.CloudupModels, ",")
 	o.SSHPublicKey = ""
 	o.OutDir = ""
-	o.CreateKubecfg = true
+	o.CreateKubecfg = false
 	o.RunTasksOptions.InitDefaults()
 }
 
@@ -115,6 +118,8 @@ func NewCmdUpdateCluster(f *util.Factory, out io.Writer) *cobra.Command {
 	cmd.Flags().StringVar(&options.SSHPublicKey, "ssh-public-key", options.SSHPublicKey, "SSH public key to use (deprecated: use kops create secret instead)")
 	cmd.Flags().StringVar(&options.OutDir, "out", options.OutDir, "Path to write any local output")
 	cmd.Flags().BoolVar(&options.CreateKubecfg, "create-kube-config", options.CreateKubecfg, "Will control automatically creating the kube config file on your local filesystem")
+	cmd.Flags().BoolVar(&options.admin, "admin", options.admin, "Also export the admin user. Implies --create-kube-config")
+	cmd.Flags().StringVar(&options.user, "user", options.user, "Existing user to add to the cluster context. Implies --create-kube-config")
 	cmd.Flags().StringVar(&options.Phase, "phase", options.Phase, "Subset of tasks to run: "+strings.Join(cloudup.Phases.List(), ", "))
 	cmd.Flags().StringSliceVar(&options.LifecycleOverrides, "lifecycle-overrides", options.LifecycleOverrides, "comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges")
 	viper.BindPFlag("lifecycle-overrides", cmd.Flags().Lookup("lifecycle-overrides"))
@@ -137,6 +142,24 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string,
 	isDryrun := false
 	targetName := c.Target
 
+	if c.admin && c.user != "" {
+		return nil, fmt.Errorf("cannot use both --admin and --user")
+	}
+
+	if c.CreateKubecfg && !c.admin && c.user == "" {
+		return nil, fmt.Errorf("--create-kube-config requires that either --admin or --user is set")
+	}
+
+	if c.admin && !c.CreateKubecfg {
+		klog.Info("--admin implies --create-kube-config")
+		c.CreateKubecfg = true
+	}
+
+	if c.user != "" && !c.CreateKubecfg {
+		klog.Info("--user implies --create-kube-config")
+		c.CreateKubecfg = true
+	}
+
 	// direct requires --yes (others do not, because they don't do anything!)
 	if c.Target == cloudup.TargetDirect {
 		if !c.Yes {
@@ -282,10 +305,11 @@ func RunUpdateCluster(ctx context.Context, f *util.Factory, clusterName string,
 		}
 		if kubecfgCert != nil {
 			klog.Infof("Exporting kubecfg for cluster")
-			conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions())
+			conf, err := kubeconfig.BuildKubecfg(cluster, keyStore, secretStore, &commands.CloudDiscoveryStatusStore{}, clientcmd.NewDefaultPathOptions(), c.admin, c.user)
 			if err != nil {
 				return nil, err
 			}
+
 			err = conf.WriteKubecfg()
 			if err != nil {
 				return nil, err
diff --git a/docs/cli/kops_export_kubecfg.md b/docs/cli/kops_export_kubecfg.md
index 1897109974..7f4fb4eb3d 100644
--- a/docs/cli/kops_export_kubecfg.md
+++ b/docs/cli/kops_export_kubecfg.md
@@ -7,7 +7,7 @@ Export kubecfg.
 
 ### Synopsis
 
-Export a kubecfg file for a cluster from the state store. The configuration will be saved into a users $HOME/.kube/config file. To export the kubectl configuration to a specific file set the KUBECONFIG environment variable.
+Export a kubecfg file for a cluster from the state store. By default the configuration will be saved into a users $HOME/.kube/config file. Kops will respect the KUBECONFIG environment variable if the --kubeconfig flag is not set.
 
 ```
 kops export kubecfg CLUSTERNAME [flags]
@@ -16,16 +16,21 @@ kops export kubecfg CLUSTERNAME [flags]
 ### Examples
 
 ```
-  # export a kubecfg file
-  kops export kubecfg kubernetes-cluster.example.com
+  # export a kubeconfig file with the cluster admin user (make sure you keep this user safe!)
+  kops export kubecfg kubernetes-cluster.example.com --admin
+  
+  # export using a user already existing in the kubeconfig file
+  kops export kubecfg kubernetes-cluster.example.com --user my-oidc-user
 ```
 
 ### Options
 
 ```
+      --admin               export the cluster admin user and add it to the context
       --all                 export all clusters from the kops state store
   -h, --help                help for kubecfg
-      --kubeconfig string   The location of the kubeconfig file to create.
+      --kubeconfig string   the location of the kubeconfig file to create.
+      --user string         add an existing user to the cluster context
 ```
 
 ### Options inherited from parent commands
diff --git a/docs/cli/kops_update_cluster.md b/docs/cli/kops_update_cluster.md
index 2c24992669..b5ed7bdf60 100644
--- a/docs/cli/kops_update_cluster.md
+++ b/docs/cli/kops_update_cluster.md
@@ -19,13 +19,14 @@ kops update cluster [flags]
 
 ```
   # After cluster has been edited or upgraded, configure it with:
-  kops update cluster k8s-cluster.example.com --yes --state=s3://kops-state-1234 --yes
+  kops update cluster k8s-cluster.example.com --yes --state=s3://kops-state-1234 --yes --admin
 ```
 
 ### Options
 
 ```
-      --create-kube-config            Will control automatically creating the kube config file on your local filesystem (default true)
+      --admin                         Also export the admin user. Implies --create-kube-config
+      --create-kube-config            Will control automatically creating the kube config file on your local filesystem
   -h, --help                          help for cluster
       --lifecycle-overrides strings   comma separated list of phase overrides, example: SecurityGroups=Ignore,InternetGateway=ExistsAndWarnIfChanges
       --model string                  Models to apply (separate multiple models with commas) (default "proto,cloudup")
@@ -33,6 +34,7 @@ kops update cluster [flags]
       --phase string                  Subset of tasks to run: assets, cluster, network, security
       --ssh-public-key string         SSH public key to use (deprecated: use kops create secret instead)
       --target string                 Target - direct, terraform, cloudformation (default "direct")
+      --user string                   Existing user to add to the cluster context. Implies --create-kube-config
   -y, --yes                           Create cloud resources, without --yes update is in dry run mode
 ```
 
diff --git a/docs/releases/1.19-NOTES.md b/docs/releases/1.19-NOTES.md
index d51f70dcd8..92010dc0d6 100644
--- a/docs/releases/1.19-NOTES.md
+++ b/docs/releases/1.19-NOTES.md
@@ -4,6 +4,16 @@
 
 # Significant changes
 
+## Changes to kubernetes config export
+
+Kops will no longer automatically export the kubernetes config on `kops update cluster`. In order to export the config on cluster update, you need to either add the `--user <user>` to reference an existing user, or `--admin` to export the cluster admin user. If neither flag is passed, the kubernetes config will not be modified. This makes it easier to reuse user definitions across clusters should you, for example, use OIDC for authentication.
+
+Similarly, `kops export kubecfg` will also require passing either the `--admin` or `--user` flag if the context does not already exist.
+
+`kops create cluster --yes` exports the admin user along with rest of the cluster config, as is existing behaviour.
+
+## Other significant changes
+
 * On AWS kops now defaults to using launch templates instead of launch configurations.
 
 * Clusters using the Amazon VPC CNI provider now perform an `ec2.DescribeInstanceTypes` call at instance launch time. In large clusters or AWS accounts this may lead to API throttling which could delay node readiness. If this becomes a problem please open a GitHub issue.
diff --git a/pkg/kubeconfig/create_kubecfg.go b/pkg/kubeconfig/create_kubecfg.go
index 2c4593294f..ea3873ab59 100644
--- a/pkg/kubeconfig/create_kubecfg.go
+++ b/pkg/kubeconfig/create_kubecfg.go
@@ -28,7 +28,7 @@ import (
 	"k8s.io/kops/upup/pkg/fi"
 )
 
-func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess) (*KubeconfigBuilder, error) {
+func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.SecretStore, status kops.StatusStore, configAccess clientcmd.ConfigAccess, admin bool, user string) (*KubeconfigBuilder, error) {
 	clusterName := cluster.ObjectMeta.Name
 
 	master := cluster.Spec.MasterPublicName
@@ -86,6 +86,7 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se
 	b := NewKubeconfigBuilder(configAccess)
 
 	b.Context = clusterName
+	b.Server = server
 
 	// add the CA Cert to the kubeconfig only if we didn't specify a SSL cert for the LB
 	if cluster.Spec.API == nil || cluster.Spec.API.LoadBalancer == nil || cluster.Spec.API.LoadBalancer.SSLCertificate == "" {
@@ -103,27 +104,30 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se
 		}
 	}
 
-	{
-		cert, key, _, err := keyStore.FindKeypair("kubecfg")
-		if err != nil {
-			return nil, fmt.Errorf("error fetching kubecfg keypair: %v", err)
-		}
-		if cert != nil {
-			b.ClientCert, err = cert.AsBytes()
+	if admin {
+		{
+			cert, key, _, err := keyStore.FindKeypair("kubecfg")
 			if err != nil {
-				return nil, err
+				return nil, fmt.Errorf("error fetching kubecfg keypair: %v", err)
 			}
-		} else {
-			return nil, fmt.Errorf("cannot find kubecfg certificate")
-		}
-		if key != nil {
-			b.ClientKey, err = key.AsBytes()
-			if err != nil {
-				return nil, err
+			if cert != nil {
+				b.ClientCert, err = cert.AsBytes()
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				return nil, fmt.Errorf("cannot find kubecfg certificate")
+			}
+			if key != nil {
+				b.ClientKey, err = key.AsBytes()
+				if err != nil {
+					return nil, err
+				}
+			} else {
+				return nil, fmt.Errorf("cannot find kubecfg key")
 			}
-		} else {
-			return nil, fmt.Errorf("cannot find kubecfg key")
 		}
+
 	}
 
 	b.Server = server
@@ -156,5 +160,11 @@ func BuildKubecfg(cluster *kops.Cluster, keyStore fi.Keystore, secretStore fi.Se
 		}
 	}
 
+	if user == "" {
+		b.User = cluster.ObjectMeta.Name
+	} else {
+		b.User = user
+	}
+
 	return b, nil
 }
diff --git a/pkg/kubeconfig/kubecfg_builder.go b/pkg/kubeconfig/kubecfg_builder.go
index 40ad85be14..e5c3d3c21c 100644
--- a/pkg/kubeconfig/kubecfg_builder.go
+++ b/pkg/kubeconfig/kubecfg_builder.go
@@ -33,6 +33,7 @@ type KubeconfigBuilder struct {
 	Context   string
 	Namespace string
 
+	User            string
 	KubeBearerToken string
 	KubeUser        string
 	KubePassword    string
@@ -116,17 +117,7 @@ func (b *KubeconfigBuilder) WriteKubecfg() error {
 			cluster = clientcmdapi.NewCluster()
 		}
 		cluster.Server = b.Server
-
-		if b.CACert == nil {
-			// For now, we assume that the cluster has a "real" cert issued by a CA
-			cluster.InsecureSkipTLSVerify = false
-			cluster.CertificateAuthority = ""
-			cluster.CertificateAuthorityData = nil
-		} else {
-			cluster.InsecureSkipTLSVerify = false
-			cluster.CertificateAuthority = ""
-			cluster.CertificateAuthorityData = b.CACert
-		}
+		cluster.CertificateAuthorityData = b.CACert
 
 		if config.Clusters == nil {
 			config.Clusters = make(map[string]*clientcmdapi.Cluster)
@@ -134,7 +125,8 @@ func (b *KubeconfigBuilder) WriteKubecfg() error {
 		config.Clusters[b.Context] = cluster
 	}
 
-	{
+	// If the user has the same name as the context, it is the admin user
+	if b.User == b.Context {
 		authInfo := config.AuthInfos[b.Context]
 		if authInfo == nil {
 			authInfo = clientcmdapi.NewAuthInfo()
@@ -158,6 +150,10 @@ func (b *KubeconfigBuilder) WriteKubecfg() error {
 			config.AuthInfos = make(map[string]*clientcmdapi.AuthInfo)
 		}
 		config.AuthInfos[b.Context] = authInfo
+	} else if b.User != "" {
+		if config.AuthInfos[b.User] == nil {
+			return fmt.Errorf("could not find user %q", b.User)
+		}
 	}
 
 	// If we have a bearer token, also create a credential entry with basic auth
@@ -186,7 +182,9 @@ func (b *KubeconfigBuilder) WriteKubecfg() error {
 		}
 
 		context.Cluster = b.Context
-		context.AuthInfo = b.Context
+		if b.User != "" {
+			context.AuthInfo = b.User
+		}
 
 		if b.Namespace != "" {
 			context.Namespace = b.Namespace


Test patch:
diff --git a/pkg/kubeconfig/create_kubecfg_test.go b/pkg/kubeconfig/create_kubecfg_test.go
index 6c949f63f6..aa15634e9c 100644
--- a/pkg/kubeconfig/create_kubecfg_test.go
+++ b/pkg/kubeconfig/create_kubecfg_test.go
@@ -119,6 +119,8 @@ func TestBuildKubecfg(t *testing.T) {
 		secretStore  fi.SecretStore
 		status       fakeStatusStore
 		configAccess clientcmd.ConfigAccess
+		admin        bool
+		user         string
 	}
 
 	publiccluster := buildMinimalCluster("testcluster", "testcluster.test.com")
@@ -132,7 +134,7 @@ func TestBuildKubecfg(t *testing.T) {
 		wantErr bool
 	}{
 		{
-			"Test Kube Config Data For Public DNS",
+			"Test Kube Config Data For Public DNS with admin",
 			args{
 				publiccluster,
 				fakeKeyStore{
@@ -146,6 +148,8 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				fakeStatusStore{},
 				nil,
+				true,
+				"",
 			},
 			&KubeconfigBuilder{
 				Context:    "testcluster",
@@ -153,6 +157,35 @@ func TestBuildKubecfg(t *testing.T) {
 				CACert:     []byte(certData),
 				ClientCert: []byte(certData),
 				ClientKey:  []byte(privatekeyData),
+				User:       "testcluster",
+			},
+			false,
+		},
+		{
+			"Test Kube Config Data For Public DNS without admin",
+			args{
+				publiccluster,
+				fakeKeyStore{
+					FindKeypairFn: func(name string) (*pki.Certificate, *pki.PrivateKey, bool, error) {
+						return fakeCertificate(),
+							fakePrivateKey(),
+							true,
+							nil
+					},
+				},
+				nil,
+				fakeStatusStore{},
+				nil,
+				false,
+				"myuser",
+			},
+			&KubeconfigBuilder{
+				Context:    "testcluster",
+				Server:     "https://testcluster.test.com",
+				CACert:     []byte(certData),
+				ClientCert: nil,
+				ClientKey:  nil,
+				User:       "myuser",
 			},
 			false,
 		},
@@ -171,6 +204,8 @@ func TestBuildKubecfg(t *testing.T) {
 				nil,
 				fakeStatusStore{},
 				nil,
+				true,
+				"",
 			},
 			&KubeconfigBuilder{
 				Context:    "emptyMasterPublicNameCluster",
@@ -178,6 +213,7 @@ func TestBuildKubecfg(t *testing.T) {
 				CACert:     []byte(certData),
 				ClientCert: []byte(certData),
 				ClientKey:  []byte(privatekeyData),
+				User:       "emptyMasterPublicNameCluster",
 			},
 			false,
 		},
@@ -204,6 +240,8 @@ func TestBuildKubecfg(t *testing.T) {
 					},
 				},
 				nil,
+				true,
+				"",
 			},
 			&KubeconfigBuilder{
 				Context:    "testgossipcluster.k8s.local",
@@ -211,13 +249,14 @@ func TestBuildKubecfg(t *testing.T) {
 				CACert:     []byte(certData),
 				ClientCert: []byte(certData),
 				ClientKey:  []byte(privatekeyData),
+				User:       "testgossipcluster.k8s.local",
 			},
 			false,
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess)
+			got, err := BuildKubecfg(tt.args.cluster, tt.args.keyStore, tt.args.secretStore, tt.args.status, tt.args.configAccess, tt.args.admin, tt.args.user)
 			if (err != nil) != tt.wantErr {
 				t.Errorf("BuildKubecfg() error = %v, wantErr %v", err, tt.wantErr)
 				return