Repository: kubean-io/kubean
Pull request #192 (2022-10-11 08:53:30)

postback kubeconfig by playbook
Signed-off-by: bo.jiang <bo.jiang@daocloud.io>

<!--  

Thanks for sending a pull request!  Here are some tips for you:

* make sure your commit is signed off

-->

**What type of PR is this?**
/kind feature

**What this PR does / why we need it**:
postback kubeconfig by playbook.

**Which issue(s) this PR fixes**:
<!--
*Automatically closes linked issue when PR is merged.
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

**Special notes for your reviewer**:




Patch:
diff --git a/Makefile b/Makefile
index cd5d8bf25..45218ce8c 100644
--- a/Makefile
+++ b/Makefile
@@ -24,7 +24,7 @@ KUBEAN_CHART_VERSION := $(shell echo ${KUBEAN_VERSION} |sed  's/^v//g' )
 REGISTRY_SERVER_ADDRESS?=container.io
 REGISTRY_REPO?=$(REGISTRY_SERVER_ADDRESS)/kubean-ci
 
-SPRAY_TAG?="master"
+SPRAY_TAG?="latest"
 
 
 .PHONY: test
@@ -60,7 +60,7 @@ spray-job: $(SOURCES)
 	export DOCKER_CLI_EXPERIMENTAL=enabled ;\
 	! ( docker buildx ls | grep spray-job-multi-platform-builder ) && docker buildx create --use --platform=$(BUILD_ARCH) --name spray-job-multi-platform-builder ;\
 	docker buildx build \
-			--build-arg spray_tag=$(SPRAY_TAG) \
+			--build-arg SPRAY_TAG=$(SPRAY_TAG) \
 			--builder spray-job-multi-platform-builder \
 			--platform $(BUILD_ARCH) \
 			--tag $(REGISTRY_REPO)/spray-job:$(KUBEAN_IMAGE_VERSION)  \
diff --git a/README.md b/README.md
index 1fc4b6172..7069f8838 100644
--- a/README.md
+++ b/README.md
@@ -1,8 +1,10 @@
 # :seedling: KuBean
 [![main workflow](https://github.com/kubean-io/kubean/actions/workflows/auto-main-ci.yaml/badge.svg)](https://github.com/kubean-io/kubean/actions/workflows/auto-main-ci.yaml) [![codecov](https://codecov.io/gh/kubean-io/kubean/branch/main/graph/badge.svg?token=8FX807D3QQ)](https://codecov.io/gh/kubean-io/kubean) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/6263/badge)](https://bestpractices.coreinfrastructure.org/projects/6263)
 [![kubean coverage](https://raw.githubusercontent.com/dasu23/e2ecoverage/master/badges/kubean/kubeanCoverage.svg)](https://github.com/kubean-io/kubean/blob/main/doc/test/kubean_testcase.md) [![kubean coverage](https://raw.githubusercontent.com/dasu23/e2ecoverage/master/badges/kubean/kubeanCoverage2.svg)](https://github.com/kubean-io/kubean/blob/main/doc/test/kubean_testcase.md)
+
 # Introduction
-kubean is a cluster lifecycle management tool based on kubespray.
+
+kubean is a cluster lifecycle management tool based on [kubespray](https://github.com/kubernetes-sigs/kubespray).
 
 # Quick Start
 
diff --git a/artifacts/demo/kubeanClusterOps.yml b/artifacts/demo/kubeanClusterOps.yml
index b3e2b2ec2..35460fa39 100644
--- a/artifacts/demo/kubeanClusterOps.yml
+++ b/artifacts/demo/kubeanClusterOps.yml
@@ -16,5 +16,7 @@ spec:
     - actionType: playbook
       action: disable-firewalld.yml
   postHook:
+    - actionType: playbook
+      action: kubeconfig.yml
     - actionType: playbook
       action: cluster-info.yml
diff --git a/pkg/util/entrypoint/entrypoint.go b/pkg/util/entrypoint/entrypoint.go
index b224ece08..549c59af1 100644
--- a/pkg/util/entrypoint/entrypoint.go
+++ b/pkg/util/entrypoint/entrypoint.go
@@ -1,6 +1,7 @@
 package entrypoint
 
 import (
+	_ "embed"
 	"fmt"
 	"strings"
 	"text/template"
@@ -21,62 +22,13 @@ const (
 	PingPB        = "ping.yml"
 	RepoPB        = "enable-repo.yml"
 	FirewallPB    = "disable-firewalld.yml"
+	KubeconfigPB  = "kubeconfig.yml"
 	ClusterInfoPB = "cluster-info.yml"
-
-	KubeconfInstall = `
-set -x
-# postback cluster kubeconfig
-inventory_file="/conf/hosts.yml"
-first_master=` + "`" + `yq e '.all.children.kube_control_plane.hosts' $inventory_file -o y | head -n 1 | cut -f1 -d":"` + "`" + `
-first_master_ip=` + "`" + `yq e '.all.hosts.'$first_master'.ip' $inventory_file -o y` + "`" + `
-kubeconfig_name="$CLUSTER_NAME-kubeconf"
-fetch_src="/root/.kube/config"
-fetch_dest="/root/$kubeconfig_name"
-ansible -i $inventory_file $first_master -m fetch -a "src=$fetch_src dest=$fetch_dest" %s
-sed -i "s/127.0.0.1:.*/"$first_master_ip":6443/" $fetch_dest/$first_master$fetch_src
-set +e
-kubeconf_count=` + "`" + `kubectl -n kubean-system get configmap | grep $kubeconfig_name | wc -l | sed 's/ //g'` + "`" + `
-if [ "${kubeconf_count}" -gt 0 ]; then
-    kubectl -n kubean-system delete configmap $kubeconfig_name
-fi
-set -e
-kubectl -n kubean-system create configmap $kubeconfig_name --from-file=$fetch_dest/$first_master$fetch_src
-kubectl patch --type=merge kubeancluster $CLUSTER_NAME -p '{"spec": {"kubeconfRef": {"name": "'$kubeconfig_name'", "namespace": "kubean-system"}}}'
-`
-	KubeconfReset = `
-set -x
-# reset cluster kubeconfig
-kubeconfig_name="$CLUSTER_NAME-kubeconf"
-set +e
-kubeconf_count=` + "`" + `kubectl -n kubean-system get configmap | grep $kubeconfig_name | wc -l | sed 's/ //g'` + "`" + `
-if [ "${kubeconf_count}" -gt 0 ]; then
-    kubectl -n kubean-system delete configmap $kubeconfig_name
-fi
-set -e
-kubectl patch --type=merge kubeancluster $CLUSTER_NAME -p '{"spec": {"kubeconfRef": null}}'
-`
-	EntrypointTemplate = `
-#!/bin/bash
-
-set -o errexit
-set -o nounset
-set -o pipefail
-
-# preinstall
-{{ range $preCMD := .PreHookCMDs }}
-{{- $preCMD }}
-{{ end }}
-
-# run kubespray
-{{ .SprayCMD }}
-
-# postinstall
-{{ range $postCMD := .PostHookCMDs }}
-{{- $postCMD }}
-{{ end }}
-`
 )
 
+//go:embed entrypoint.sh.template
+var entrypointTemplate string
+
 type void struct{}
 
 var member void
@@ -97,7 +49,7 @@ func NewActions() *Actions {
 	actions.Playbooks = &Playbooks{}
 	actions.Playbooks.List = []string{
 		ResetPB, ScalePB, ClusterPB, RemoveNodePB, UpgradeClusterPB,
-		PingPB, RepoPB, FirewallPB, ClusterInfoPB,
+		PingPB, RepoPB, FirewallPB, KubeconfigPB, ClusterInfoPB,
 	}
 	actions.Playbooks.Dict = map[string]void{}
 	for _, pbItem := range actions.Playbooks.List {
@@ -182,30 +134,6 @@ func (ep *EntryPoint) PostHookRunPart(actionType, action, extraArgs string, isPr
 	return nil
 }
 
-func (ep *EntryPoint) kubeconfPostbackPart(action string, isPrivateKey bool) error {
-	if _, ok := ep.Actions.Playbooks.Dict[action]; !ok {
-		return nil
-	}
-	script := ""
-	if action == ClusterPB {
-		script = KubeconfInstall
-		if isPrivateKey {
-			script = fmt.Sprintf(script, "--private-key /auth/ssh-privatekey")
-		} else {
-			script = fmt.Sprintf(script, "")
-		}
-	}
-	if action == ResetPB {
-		script = KubeconfReset
-	}
-	posthook, err := ep.hookRunPart(SHAction, script, "", false)
-	if err != nil {
-		return ArgsError{fmt.Sprintf("posthook: %s", err)}
-	}
-	ep.PostHookCMDs = append(ep.PostHookCMDs, posthook)
-	return nil
-}
-
 func (ep *EntryPoint) SprayRunPart(actionType, action, extraArgs string, isPrivateKey bool) error {
 	if actionType == PBAction {
 		playbookCmd, err := ep.buildPlaybookCmd(action, extraArgs, isPrivateKey)
@@ -218,15 +146,12 @@ func (ep *EntryPoint) SprayRunPart(actionType, action, extraArgs string, isPriva
 	} else {
 		return ArgsError{fmt.Sprintf("unknown action type, the currently supported ranges include: %s", ep.Actions.Types)}
 	}
-	if err := ep.kubeconfPostbackPart(action, isPrivateKey); err != nil {
-		return ArgsError{fmt.Sprintf("failed to set kubeconfig postback: %s", err)}
-	}
 	return nil
 }
 
 func (ep *EntryPoint) Render() (string, error) {
 	b := &strings.Builder{}
-	tmpl := template.Must(template.New("entrypoint").Parse(EntrypointTemplate))
+	tmpl := template.Must(template.New("entrypoint").Parse(entrypointTemplate))
 	if err := tmpl.Execute(b, ep); err != nil {
 		return "", err
 	}
diff --git a/pkg/util/entrypoint/entrypoint.sh.template b/pkg/util/entrypoint/entrypoint.sh.template
new file mode 100644
index 000000000..bb8bea459
--- /dev/null
+++ b/pkg/util/entrypoint/entrypoint.sh.template
@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+# preinstall
+{{ range $preCMD := .PreHookCMDs }}
+{{- $preCMD }}
+{{ end }}
+
+# run kubespray
+{{ .SprayCMD }}
+
+# postinstall
+{{ range $postCMD := .PostHookCMDs }}
+{{- $postCMD }}
+{{ end }}
diff --git a/playbooks/kubeconfig.yml b/playbooks/kubeconfig.yml
new file mode 100644
index 000000000..863ae1407
--- /dev/null
+++ b/playbooks/kubeconfig.yml
@@ -0,0 +1,86 @@
+---
+- hosts: kube_control_plane[0]
+  gather_facts: no
+  tasks:
+    - name: Copy kubeconfig from master to local
+      fetch:
+        src: /root/.kube/config
+        dest: /conf/.kube/
+        flat: yes
+      register: cp_kubeconf_result
+      until: cp_kubeconf_result is not failed
+      retries: 10
+      when: not postback_undo|default(false)|bool
+
+- hosts: localhost
+  gather_facts: false
+  vars:
+    cluster_name: "{{ lookup('env','CLUSTER_NAME') }}"
+    kubeconfig_name: "{{ cluster_name }}-kubeconf"
+    local_kube_config: /conf/.kube/config
+    first_master_host: "{{ hostvars[groups['kube_control_plane'][0]].ansible_host }}"
+  tasks:
+  
+    # Install
+
+    - name: Update kubeconfig server
+      shell: "sed -i 's/127.0.0.1:.*/{{ first_master_host }}:6443/' {{ local_kube_config }}"
+      args:
+        executable: /bin/bash
+      register: update_kubeconfig_result
+      failed_when: update_kubeconfig_result.stderr
+      when: not postback_undo|default(false)|bool
+
+    - name: Check kubeconfig configmap
+      shell: "kubectl -n kubean-system get configmap {{ kubeconfig_name }}"
+      args:
+        executable: /bin/bash
+      register: kubeconfig_cm
+      ignore_errors: true
+
+    - name: Delete kubeconfig configmap
+      shell: "kubectl -n kubean-system delete configmap {{ kubeconfig_name }}"
+      args:
+        executable: /bin/bash
+      when: kubeconfig_cm.rc == 0
+
+    - name: Create kubeconfig configmap
+      shell: "kubectl -n kubean-system create configmap {{ kubeconfig_name }} --from-file={{ local_kube_config }}"
+      args:
+        executable: /bin/bash
+      register: create_cm_result
+      until: create_cm_result.rc == 0
+      retries: 5
+      delay: 5
+      when: not postback_undo|default(false)|bool
+
+    - name: Patch kubeconfRef info
+      shell: >
+        kubectl patch --type=merge kubeancluster {{ cluster_name }} -p '{"spec": {"kubeconfRef": {"name": "'{{ kubeconfig_name }}'", "namespace": "kubean-system"}}}'
+      args:
+        executable: /bin/bash
+      register: patch_ref_result
+      until: patch_ref_result.rc == 0
+      retries: 5
+      delay: 5
+      when: not postback_undo|default(false)|bool
+
+    # Uninstall
+
+    - name: Clean kubeconfRef info
+      shell: >
+        kubectl patch --type=merge kubeancluster {{ cluster_name }} -p '{"spec": {"kubeconfRef": null}}'
+      args:
+        executable: /bin/bash
+      register: clean_ref_result
+      until: clean_ref_result.rc == 0
+      retries: 5
+      delay: 5
+      when: postback_undo|default(false)|bool
+
+    - name: Delete local kubeconfig file
+      shell: "rm -rf {{ local_kube_config }}"
+      args:
+        executable: /bin/bash
+      ignore_errors: true
+      when: postback_undo|default(false)|bool


Test patch:
diff --git a/pkg/util/entrypoint/entrypoint_test.go b/pkg/util/entrypoint/entrypoint_test.go
index c79477e24..d03d6da0a 100644
--- a/pkg/util/entrypoint/entrypoint_test.go
+++ b/pkg/util/entrypoint/entrypoint_test.go
@@ -104,21 +104,21 @@ var testData = `
   input:
     actionType: playbook
     action: remove-node.yml
-  matchString: "remove-node.yml"
+  matchString: "-e \"skip_confirmation=true\" /kubespray/remove-node.yml"
   output: true
 
 - message: "Check kubespray upgrade-cluster"
   input:
     actionType: playbook
     action: upgrade-cluster.yml
-  matchString: "upgrade-cluster.yml"
+  matchString: "/kubespray/upgrade-cluster.yml"
   output: true
 
 - message: "Check kubespray scale"
   input:
     actionType: playbook
     action: scale.yml
-  matchString: "scale.yml"
+  matchString: "/kubespray/scale.yml"
   output: true
 
 - message: "Check scale parameter 'extraArgs'"
@@ -126,7 +126,7 @@ var testData = `
     actionType: playbook
     action: scale.yml
     extraArgs: "--limit=node3,node4"
-  matchString: "--limit=node3,node4"
+  matchString: "/kubespray/scale.yml --limit=node3,node4"
   output: true
 
 - message: "Check remove node parameter 'extraArgs'"
@@ -134,22 +134,23 @@ var testData = `
     actionType: playbook
     action: remove-node.yml
     extraArgs: "-e node=node3"
-  matchString: "-e node=node3"
+  matchString: "-e \"skip_confirmation=true\" /kubespray/remove-node.yml -e node=node3"
   output: true
 
 - message: "Check postback kubeconf when the cluster is installed"
   input:
     actionType: playbook
-    action: cluster.yml
+    action: kubeconfig.yml
     isPrivateKey: true
-  matchString: "ansible -i $inventory_file $first_master -m fetch"
+  matchString: "--private-key /auth/ssh-privatekey /kubespray/kubeconfig.yml"
   output: true
 
 - message: "Check clean up kubeconf after cluster reset"
   input:
     actionType: playbook
-    action: reset.yml
-  matchString: "{\"spec\": {\"kubeconfRef\": null}}"
+    action: kubeconfig.yml
+    extraArgs: "-e postback_undo=true"
+  matchString: "/kubespray/kubeconfig.yml -e postback_undo=true"
   output: true
 
 - message: "Check entrypoint script cmd concatenation"