Repository: kubeclipper/kubeclipper
Pull request #658 (2023-08-24 06:46:13)

support set k8s featureGates
<!-- Thanks for sending a pull request! Here are some tips for you:

1. If you want **faster** PR reviews, read how: https://github.com/kubeclipper/community/blob/master/developer-guide/development/the-pr-author-guide-to-getting-through-code-review.md
2. In case you want to know how your PR got reviewed, read: https://github.com/kubeclipper/community/blob/master/developer-guide/development/code-review-guide.md
3. Here are some coding convetions followed by Kubeclipper community: https://github.com/kubeclipper/community/blob/master/developer-guide/development/coding-conventions.md
-->

### What type of PR is this?
<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->

/kind feature
### What this PR does / why we need it:

### Which issue(s) this PR fixes:
<!--
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

### Special notes for reviewers:
```
```

### Does this PR introduced a user-facing change?
<!--
If no, just write "None" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://github.com/kubernetes/community/blob/master/contributors/guide/release-notes.md
-->
```release-note
none
```

### Additional documentation, usage docs, etc.:
<!--
This section can be blank if this pull request does not require a release note.
Please use the following format for linking documentation or pass the
section below:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```

kubeclipper-bot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubeclipper/kubeclipper/pull/658#" title="Author self-approved">x893675</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubeclipper%2Fkubeclipper).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/kubeclipper/kubeclipper/blob/master/OWNERS)~~ [x893675]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/pkg/cli/create/create.go b/pkg/cli/create/create.go
index 44f924d..f1fdd1b 100644
--- a/pkg/cli/create/create.go
+++ b/pkg/cli/create/create.go
@@ -78,6 +78,8 @@ masters:
 #        effect: NoSchedule
 workers: []
 certSans: []
+#featureGates:
+#  xxx: true
 cni:
   calico:
     IPManger: true
diff --git a/pkg/cli/create/create_cluster.go b/pkg/cli/create/create_cluster.go
index f84a188..a52d2c2 100644
--- a/pkg/cli/create/create_cluster.go
+++ b/pkg/cli/create/create_cluster.go
@@ -25,6 +25,7 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"strconv"
 	"strings"
 
 	"github.com/kubeclipper/kubeclipper/pkg/constatns"
@@ -135,6 +136,8 @@ type CreateClusterOptions struct {
 	ServiceSubnet             string
 	PodSubnet                 string
 	OnlyInstallKubernetesComp bool
+	FeatureGatesString        []string
+	FeatureGates              map[string]bool
 }
 
 var (
@@ -158,6 +161,7 @@ func NewCreateClusterOptions(streams options.IOStreams) *CreateClusterOptions {
 		IPv4AutoDetection: autodetection.MethodFirst,
 		ServiceSubnet:     "10.96.0.0/12",
 		PodSubnet:         "172.25.0.0/16",
+		FeatureGates:      map[string]bool{},
 	}
 }
 
@@ -197,6 +201,7 @@ func NewCmdCreateCluster(streams options.IOStreams) *cobra.Command {
 	cmd.Flags().StringVar(&o.ServiceSubnet, "service-subnet", o.ServiceSubnet, "serviceSubnet is the subnet used by Kubernetes Services. Defaults to '10.96.0.0/12'")
 	cmd.Flags().StringVar(&o.PodSubnet, "pod-subnet", o.PodSubnet, "podSubnet is the subnet used by Pods. Defaults to '172.25.0.0/16'")
 	cmd.Flags().BoolVar(&o.OnlyInstallKubernetesComp, "only-install-kubernetes-component", o.OnlyInstallKubernetesComp, "only install kubernetes component, not install cni")
+	cmd.Flags().StringSliceVar(&o.FeatureGatesString, "feature-gates", o.FeatureGatesString, "k8s feature gates, format as: --feature-gates=xxx=true|false")
 	o.CliOpts.AddFlags(cmd.Flags())
 	o.PrintFlags.AddFlags(cmd)
 
@@ -330,6 +335,23 @@ func (l *CreateClusterOptions) ValidateArgs(cmd *cobra.Command) error {
 		l.CaCertFile = base64.StdEncoding.EncodeToString(caCert)
 		l.CaKeyFile = base64.StdEncoding.EncodeToString(caKey)
 	}
+
+	for _, fg := range l.FeatureGatesString {
+		if strings.Contains(fg, "=") {
+			arr := strings.Split(fg, "=")
+			if len(arr) != 2 {
+				continue
+			}
+			k := strings.TrimSpace(arr[0])
+			v := strings.TrimSpace(arr[1])
+			boolValue, err := strconv.ParseBool(v)
+			if err != nil {
+				return fmt.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k)
+			}
+			l.FeatureGates[k] = boolValue
+		}
+	}
+
 	return nil
 }
 
@@ -432,6 +454,7 @@ func (l *CreateClusterOptions) newCluster() *v1.Cluster {
 				MTU:               1440,
 			},
 		},
+		FeatureGates: l.FeatureGates,
 
 		Status: v1.ClusterStatus{},
 	}
diff --git a/pkg/scheme/core/v1/cluster_types.go b/pkg/scheme/core/v1/cluster_types.go
index 500e075..e65cee6 100644
--- a/pkg/scheme/core/v1/cluster_types.go
+++ b/pkg/scheme/core/v1/cluster_types.go
@@ -57,6 +57,7 @@ type Cluster struct {
 	Description       string             `json:"description,omitempty" optional:"true"`
 	Status            ClusterStatus      `json:"status,omitempty" optional:"true"`
 	PendingOperations []PendingOperation `json:"pendingOperations,omitempty" optional:"true"`
+	FeatureGates      map[string]bool    `json:"featureGates,omitempty"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
diff --git a/pkg/scheme/core/v1/k8s/cluster.go b/pkg/scheme/core/v1/k8s/cluster.go
index 21c7791..97afffe 100644
--- a/pkg/scheme/core/v1/k8s/cluster.go
+++ b/pkg/scheme/core/v1/k8s/cluster.go
@@ -149,6 +149,7 @@ func (stepper *Upgrade) InitStepper(metadata *component.ExtraMetadata, c *v1.Clu
 		ControlPlaneEndpoint:    cpEndpoint,
 		CertSANs:                c.GetAllCertSANs(),
 		LocalRegistry:           c.LocalRegistry,
+		FeatureGates:            c.FeatureGates,
 	}
 	stepper.Offline = metadata.Offline
 	stepper.Version = metadata.KubeVersion
diff --git a/pkg/scheme/core/v1/k8s/kubeadm.go b/pkg/scheme/core/v1/k8s/kubeadm.go
index bc6ca23..6713920 100644
--- a/pkg/scheme/core/v1/k8s/kubeadm.go
+++ b/pkg/scheme/core/v1/k8s/kubeadm.go
@@ -105,22 +105,23 @@ type KubeadmConfig struct {
 	ClusterConfigAPIVersion string `json:"clusterConfigAPIVersion"`
 	// If both Docker and containerd are detected, Docker takes precedence,so we must specify cri.
 	// https://v1-20.docs.kubernetes.io/docs/setup/production-environment/tools/kubeadm/install-kubeadm/#installing-runtime
-	ContainerRuntime     string        `json:"containerRuntime"`
-	Etcd                 v1.Etcd       `json:"etcd"`
-	Networking           v1.Networking `json:"networking"`
-	KubeProxy            v1.KubeProxy  `json:"kubeProxy"`
-	Kubelet              v1.Kubelet    `json:"kubelet"`
-	ClusterName          string        `json:"clusterName"`
-	KubernetesVersion    string        `json:"kubernetesVersion"`
-	ControlPlaneEndpoint string        `json:"controlPlaneEndpoint"`
-	CertSANs             []string      `json:"certSANs"`
-	LocalRegistry        string        `json:"localRegistry"`
-	Offline              bool          `json:"offline"`
-	IsControlPlane       bool          `json:"isControlPlane,omitempty"`
-	CACertHashes         string        `json:"caCertHashes,omitempty"`
-	BootstrapToken       string        `json:"bootstrapToken,omitempty"`
-	CertificateKey       string        `json:"certificateKey,omitempty"`
-	AdvertiseAddress     string        `json:"advertiseAddress,omitempty"`
+	ContainerRuntime     string          `json:"containerRuntime"`
+	Etcd                 v1.Etcd         `json:"etcd"`
+	Networking           v1.Networking   `json:"networking"`
+	KubeProxy            v1.KubeProxy    `json:"kubeProxy"`
+	Kubelet              v1.Kubelet      `json:"kubelet"`
+	ClusterName          string          `json:"clusterName"`
+	KubernetesVersion    string          `json:"kubernetesVersion"`
+	ControlPlaneEndpoint string          `json:"controlPlaneEndpoint"`
+	CertSANs             []string        `json:"certSANs"`
+	LocalRegistry        string          `json:"localRegistry"`
+	Offline              bool            `json:"offline"`
+	IsControlPlane       bool            `json:"isControlPlane,omitempty"`
+	CACertHashes         string          `json:"caCertHashes,omitempty"`
+	BootstrapToken       string          `json:"bootstrapToken,omitempty"`
+	CertificateKey       string          `json:"certificateKey,omitempty"`
+	AdvertiseAddress     string          `json:"advertiseAddress,omitempty"`
+	FeatureGates         map[string]bool `json:"featureGates,omitempty"`
 }
 
 type ControlPlane struct {
diff --git a/pkg/scheme/core/v1/k8s/kubeadm_step.go b/pkg/scheme/core/v1/k8s/kubeadm_step.go
index 40f3a04..4318472 100644
--- a/pkg/scheme/core/v1/k8s/kubeadm_step.go
+++ b/pkg/scheme/core/v1/k8s/kubeadm_step.go
@@ -430,6 +430,7 @@ func (stepper *KubeadmConfig) InitStepper(c *v1.Cluster, metadata *component.Ext
 	stepper.CertSANs = c.GetAllCertSANs()
 	stepper.LocalRegistry = c.LocalRegistry
 	stepper.Offline = metadata.Offline
+	stepper.FeatureGates = c.FeatureGates
 	// TODO: No vip is currently introduced as controlPlaneEndpoint
 	stepper.AdvertiseAddress = metadata.Masters[0].NodeIPv4
 
diff --git a/pkg/scheme/core/v1/k8s/template.go b/pkg/scheme/core/v1/k8s/template.go
index 3fddda9..c77bff3 100644
--- a/pkg/scheme/core/v1/k8s/template.go
+++ b/pkg/scheme/core/v1/k8s/template.go
@@ -61,6 +61,9 @@ scheduler:
     pathType: File
 {{with .LocalRegistry}}imageRepository: {{.}}{{end}}
 {{with .ClusterName}}clusterName: {{.}}{{end}}
+{{if gt (len .FeatureGates) 0}}
+featureGates:{{range $key,$value := .FeatureGates}}
+  {{$key}}: {{$value}}{{end}}{{end}}
 ---
 kind: KubeProxyConfiguration
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
diff --git a/pkg/scheme/core/v1/zz_generated.deepcopy.go b/pkg/scheme/core/v1/zz_generated.deepcopy.go
index c866936..e1dbc69 100644
--- a/pkg/scheme/core/v1/zz_generated.deepcopy.go
+++ b/pkg/scheme/core/v1/zz_generated.deepcopy.go
@@ -439,6 +439,13 @@ func (in *Cluster) DeepCopyInto(out *Cluster) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.FeatureGates != nil {
+		in, out := &in.FeatureGates, &out.FeatureGates
+		*out = make(map[string]bool, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	return
 }
 


Test patch:
diff --git a/pkg/scheme/core/v1/k8s/kubeadm_test.go b/pkg/scheme/core/v1/k8s/kubeadm_test.go
index 8f64711..94562cf 100644
--- a/pkg/scheme/core/v1/k8s/kubeadm_test.go
+++ b/pkg/scheme/core/v1/k8s/kubeadm_test.go
@@ -52,6 +52,7 @@ func TestKubeadmConfig_renderTo(t *testing.T) {
 		ControlPlaneEndpoint    string
 		CertSANs                []string
 		LocalRegistry           string
+		FeatureGates            map[string]bool
 	}
 	tests := []struct {
 		name   string
@@ -104,6 +105,33 @@ func TestKubeadmConfig_renderTo(t *testing.T) {
 				LocalRegistry:        "127.0.0.1:5000",
 			},
 		},
+		{
+			name: "add featureGates",
+			fields: fields{
+				ClusterConfigAPIVersion: "v1beta3",
+				ContainerRuntime:        "containerd",
+				Etcd:                    v1.Etcd{DataDir: "/var/lib/etcd"},
+				Networking: v1.Networking{
+					IPFamily:      v1.IPFamilyIPv4,
+					Services:      v1.NetworkRanges{CIDRBlocks: []string{constatns.ClusterServiceSubnet}},
+					Pods:          v1.NetworkRanges{CIDRBlocks: []string{constatns.ClusterPodSubnet}},
+					DNSDomain:     "cluster.local",
+					ProxyMode:     "ipvs",
+					WorkerNodeVip: "8.8.8.8",
+				},
+				KubeProxy:            v1.KubeProxy{},
+				Kubelet:              v1.Kubelet{RootDir: "/var/lib/kubelet", NodeIP: "127.0.0.1", IPAsName: true},
+				ClusterName:          "test-cluster",
+				KubernetesVersion:    "v1.23.6",
+				ControlPlaneEndpoint: "apiserver.cluster.local:6443",
+				CertSANs:             []string{"127.0.0.1"},
+				LocalRegistry:        "127.0.0.1:5000",
+				FeatureGates: map[string]bool{
+					"AdmissionWebhookMatchConditions": true,
+					"AggregatedDiscoveryEndpoint":     true,
+				},
+			},
+		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
@@ -119,6 +147,7 @@ func TestKubeadmConfig_renderTo(t *testing.T) {
 				ControlPlaneEndpoint:    tt.fields.ControlPlaneEndpoint,
 				CertSANs:                tt.fields.CertSANs,
 				LocalRegistry:           tt.fields.LocalRegistry,
+				FeatureGates:            tt.fields.FeatureGates,
 			}
 			w := &bytes.Buffer{}
 			err := stepper.renderTo(w)