Repository: kubeclipper/kubeclipper
Pull request #450 (2022-11-29 09:26:37)

feat: kc node info and detail e2e test
Signed-off-by: xu.jingwen <xu.jingwen@99cloud.net>

<!-- Thanks for sending a pull request! Here are some tips for you:

1. If you want **faster** PR reviews, read how: https://github.com/kubeclipper/community/blob/master/developer-guide/development/the-pr-author-guide-to-getting-through-code-review.md
2. In case you want to know how your PR got reviewed, read: https://github.com/kubeclipper/community/blob/master/developer-guide/development/code-review-guide.md
3. Here are some coding convetions followed by Kubeclipper community: https://github.com/kubeclipper/community/blob/master/developer-guide/development/coding-conventions.md
-->

### What type of PR is this?
/kind flake
<!--
Add one of the following kinds:
/kind bug
/kind cleanup
/kind documentation
/kind feature
/kind design

Optionally add one or more of the following kinds if applicable:
/kind api-change
/kind deprecation
/kind failing-test
/kind flake
/kind regression
-->


### What this PR does / why we need it:

### Which issue(s) this PR fixes:
<!--
Usage: `Fixes #<issue number>`, or `Fixes (paste link of issue)`.
_If PR is about `failing-tests or flakes`, please post the related issues/tests in a comment and do not use `Fixes`_*
-->
Fixes #

### Special notes for reviewers:
```
1. node fuzzy search and node detail
2. node e2e structure modify
```

### Does this PR introduced a user-facing change?
<!--
If no, just write "None" in the release-note block below.
If yes, a release note is required:
Enter your extended release note in the block below. If the PR requires additional action from users switching to the new release, include the string "action required".

For more information on release notes see: https://github.com/kubernetes/community/blob/master/contributors/guide/release-notes.md
-->
```release-note

```

### Additional documentation, usage docs, etc.:
<!--
This section can be blank if this pull request does not require a release note.
Please use the following format for linking documentation or pass the
section below:
- [KEP]: <link>
- [Usage]: <link>
- [Other doc]: <link>
-->
```docs

```

kubeclipper-bot: @Xvv-v: Adding the "do-not-merge/release-note-label-needed" label because no release-note block was detected, please follow our [release note process](https://git.k8s.io/community/contributors/guide/release-notes.md) to remove it.

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository.
</details>
kubeclipper-bot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/kubeclipper/kubeclipper/pull/450#" title="Author self-approved">Xvv-v</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **x893675** after the PR has been reviewed.
You can assign the PR to them by writing `/assign @x893675` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=kubeclipper%2Fkubeclipper).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/kubeclipper/kubeclipper/blob/release-1.3/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["x893675"]} -->

Patch:
diff --git a/pkg/simple/client/kc/api.go b/pkg/simple/client/kc/api.go
index dad9f2e..e2cd62a 100644
--- a/pkg/simple/client/kc/api.go
+++ b/pkg/simple/client/kc/api.go
@@ -49,6 +49,7 @@ const (
 	projectPath          = "/api/tenant.kubeclipper.io/v1/projects"
 	templatePath         = "/api/core.kubeclipper.io/v1/templates"
 	registryPath         = "/api/core.kubeclipper.io/v1/registries"
+	regionPath           = "/api/core.kubeclipper.io/v1/regions"
 )
 
 func (cli *Client) ListNodes(ctx context.Context, query Queries) (*NodesList, error) {
@@ -603,3 +604,14 @@ func (cli *Client) ListRegistries(ctx context.Context, query Queries) (*v1.Regis
 	err = json.NewDecoder(serverResp.body).Decode(&registries)
 	return &registries, err
 }
+
+func (cli *Client) ListRegion(ctx context.Context, query Queries) (*v1.RegionList, error) {
+	serverResp, err := cli.get(ctx, regionPath, query.ToRawQuery(), nil)
+	defer ensureReaderClosed(serverResp)
+	if err != nil {
+		return nil, err
+	}
+	regions := v1.RegionList{}
+	err = json.NewDecoder(serverResp.body).Decode(&regions)
+	return &regions, err
+}
diff --git a/pkg/simple/client/kc/request.go b/pkg/simple/client/kc/request.go
index d26e12c..bf79efc 100644
--- a/pkg/simple/client/kc/request.go
+++ b/pkg/simple/client/kc/request.go
@@ -59,6 +59,14 @@ func (q *Queries) ToRawQuery() url.Values {
 		queryParameters.Set(query.ParameterFieldSelector, q.FieldSelector)
 	}
 
+	if len(q.FuzzySearch) > 0 {
+		var fuzzy string
+		for k, v := range q.FuzzySearch {
+			fuzzy += fmt.Sprintf("%s~%s", k, v)
+		}
+		queryParameters.Set(query.ParameterFuzzySearch, fuzzy)
+	}
+
 	return queryParameters
 }
 


Test patch:
diff --git a/pkg/simple/client/kc/request_test.go b/pkg/simple/client/kc/request_test.go
new file mode 100644
index 0000000..d755928
--- /dev/null
+++ b/pkg/simple/client/kc/request_test.go
@@ -0,0 +1,33 @@
+package kc
+
+import (
+	"net/url"
+	"testing"
+
+	"github.com/kubeclipper/kubeclipper/pkg/query"
+)
+
+func TestQueries_ToRawQuery(t *testing.T) {
+	q := query.New()
+	q.LabelSelector = "a=b"
+	q.FieldSelector = "metadata.name=demo"
+	q.Watch = true
+	q.FuzzySearch = map[string]string{
+		"name": "demo",
+	}
+	kcq := Queries(*q)
+	rawQuery := kcq.ToRawQuery()
+	got := rawQuery.Encode()
+
+	target := make(url.Values)
+	target.Set("paging", "limit=-1,page=0")
+	target.Set("watch", "true")
+	target.Set("fieldSelector", "metadata.name=demo")
+	target.Set("labelSelector", "a=b")
+	target.Set("fuzzy", "name~demo")
+	want := target.Encode()
+
+	if got != want {
+		t.Fatalf("want:%s got:%s\n", want, got)
+	}
+}
diff --git a/test/e2e/cluster/spec.go b/test/e2e/cluster/spec.go
index 0991c5c..8270d93 100644
--- a/test/e2e/cluster/spec.go
+++ b/test/e2e/cluster/spec.go
@@ -236,7 +236,9 @@ var _ = SIGDescribe("[Serial]", func() {
 		framework.ExpectNoError(err)
 
 		ginkgo.By("delete registry")
-		err = f.Client.DeleteRegistry(context.TODO(), kcRegistry)
+		err = retryOperation(func() error {
+			return f.Client.DeleteRegistry(context.TODO(), kcRegistry)
+		}, 2)
 		framework.ExpectNoError(err)
 	})
 
diff --git a/test/e2e/cluster/utils.go b/test/e2e/cluster/utils.go
index fc2156c..3894c77 100644
--- a/test/e2e/cluster/utils.go
+++ b/test/e2e/cluster/utils.go
@@ -21,6 +21,7 @@ package cluster
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/onsi/ginkgo"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -183,14 +184,30 @@ func beforeEachCreateCluster(f *framework.Framework, c *corev1.Cluster) func() {
 func afterEachDeleteCluster(f *framework.Framework, clusterName *string) func() {
 	return func() {
 		ginkgo.By("delete cluster")
-		err := f.Client.DeleteCluster(context.TODO(), *clusterName)
-		if apierror.IsNotFound(err) {
-			// cluster not exist
-			return
-		}
+		err := retryOperation(func() error {
+			delErr := f.Client.DeleteCluster(context.TODO(), *clusterName)
+			if apierror.IsNotFound(delErr) {
+				// cluster not exist
+				return nil
+			}
+			return delErr
+		}, 2)
 		framework.ExpectNoError(err)
 		ginkgo.By("waiting for cluster to be deleted")
 		err = cluster.WaitForClusterNotFound(f.Client, *clusterName, f.Timeouts.ClusterDelete)
 		framework.ExpectNoError(err)
 	}
 }
+
+type fn func() error
+
+func retryOperation(f fn, times int) error {
+	var err error
+	for i := 0; i < times; i++ {
+		if err = f(); err == nil {
+			break
+		}
+		time.Sleep(5 * time.Second)
+	}
+	return err
+}
diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go
index 0dd660b..d7caee6 100644
--- a/test/e2e/e2e_test.go
+++ b/test/e2e/e2e_test.go
@@ -27,6 +27,7 @@ import (
 
 	_ "github.com/kubeclipper/kubeclipper/test/e2e/cluster"
 	_ "github.com/kubeclipper/kubeclipper/test/e2e/node"
+	_ "github.com/kubeclipper/kubeclipper/test/e2e/region"
 	"github.com/kubeclipper/kubeclipper/test/framework"
 )
 
diff --git a/test/e2e/node/drain_node.go b/test/e2e/kcctl/drain_node.go
similarity index 71%
rename from test/e2e/node/drain_node.go
rename to test/e2e/kcctl/drain_node.go
index 216eb79..eebbdac 100644
--- a/test/e2e/node/drain_node.go
+++ b/test/e2e/kcctl/drain_node.go
@@ -1,16 +1,15 @@
-package node
+package kcctl
 
 import (
 	"context"
-	"fmt"
 
 	"github.com/onsi/ginkgo"
 
 	"github.com/kubeclipper/kubeclipper/pkg/query"
 	"github.com/kubeclipper/kubeclipper/pkg/simple/client/kc"
-	"github.com/kubeclipper/kubeclipper/pkg/utils/cmdutil"
 	"github.com/kubeclipper/kubeclipper/test/framework"
 	"github.com/kubeclipper/kubeclipper/test/framework/cluster"
+	fnode "github.com/kubeclipper/kubeclipper/test/framework/node"
 )
 
 var _ = SIGDescribe("[Fast] [Serial] Force drain node", func() {
@@ -43,24 +42,11 @@ var _ = SIGDescribe("[Fast] [Serial] Force drain node", func() {
 
 	ginkgo.It("force drain agent node and ensure the node is deleted", func() {
 		ginkgo.By("drain agent node")
-		err := drainAgentNode(nodeID)
+		err := drainAgentNodeForce(nodeID)
 		framework.ExpectNoError(err)
 
 		ginkgo.By("waiting for node is deleted")
-		err = cluster.WaitForNodeNotFound(f.Client, nodeID, f.Timeouts.CommonTimeout)
+		err = fnode.WaitForNodeNotFound(f.Client, nodeID, f.Timeouts.CommonTimeout)
 		framework.ExpectNoError(err)
 	})
 })
-
-func drainAgentNode(nodeID string) error {
-	return runCmd(fmt.Sprintf("echo y | kcctl drain --agent %s -F", nodeID))
-}
-
-func joinAgentNode(nodeIP string) error {
-	return runCmd(fmt.Sprintf("kcctl join --agent %s", nodeIP))
-}
-
-func runCmd(cmd string) error {
-	ec := cmdutil.NewExecCmd(context.TODO(), "bash", "-c", cmd)
-	return ec.Run()
-}
diff --git a/test/e2e/kcctl/framework.go b/test/e2e/kcctl/framework.go
new file mode 100644
index 0000000..df335e7
--- /dev/null
+++ b/test/e2e/kcctl/framework.go
@@ -0,0 +1,26 @@
+/*
+ *
+ *  * Copyright 2021 KubeClipper Authors.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *     http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package kcctl
+
+import "github.com/onsi/ginkgo"
+
+// SIGDescribe annotates the test with the SIG label.
+func SIGDescribe(text string, body func()) bool {
+	return ginkgo.Describe("[sig-kcctl] "+text, body)
+}
diff --git a/test/e2e/node/join_node.go b/test/e2e/kcctl/join_node.go
similarity index 86%
rename from test/e2e/node/join_node.go
rename to test/e2e/kcctl/join_node.go
index a0ae8b3..de06f42 100644
--- a/test/e2e/node/join_node.go
+++ b/test/e2e/kcctl/join_node.go
@@ -16,7 +16,7 @@
  *
  */
 
-package node
+package kcctl
 
 import (
 	"context"
@@ -26,11 +26,11 @@ import (
 	"github.com/kubeclipper/kubeclipper/pkg/scheme/common"
 	"github.com/kubeclipper/kubeclipper/pkg/simple/client/kc"
 	"github.com/kubeclipper/kubeclipper/test/e2e/cluster"
+	"github.com/kubeclipper/kubeclipper/test/framework/node"
 
 	"github.com/onsi/ginkgo"
 
 	"github.com/kubeclipper/kubeclipper/test/framework"
-	fcluster "github.com/kubeclipper/kubeclipper/test/framework/cluster"
 )
 
 var _ = SIGDescribe("[Medium] [Serial] Join node", func() {
@@ -54,11 +54,11 @@ var _ = SIGDescribe("[Medium] [Serial] Join node", func() {
 
 	ginkgo.It("join node", func() {
 		ginkgo.By("drain node")
-		err := drainNode(nodeID)
+		err := drainAgentNode(nodeID)
 		framework.ExpectNoError(err)
 
 		ginkgo.By("wait for node not found")
-		err = fcluster.WaitForNodeNotFound(f.Client, nodeIP, f.Timeouts.CommonTimeout)
+		err = node.WaitForNodeNotFound(f.Client, nodeIP, f.Timeouts.CommonTimeout)
 		framework.ExpectNoError(err)
 		framework.Logf("node %s drained\n", nodeIP)
 
@@ -67,7 +67,7 @@ var _ = SIGDescribe("[Medium] [Serial] Join node", func() {
 		framework.ExpectNoError(err)
 
 		ginkgo.By("wait for node join")
-		nodeID, err = fcluster.WaitForNodeJoin(f.Client, nodeIP, f.Timeouts.CommonTimeout)
+		nodeID, err = node.WaitForNodeJoin(f.Client, nodeIP, f.Timeouts.CommonTimeout)
 		framework.ExpectNoError(err)
 		framework.Logf("node %s registered，id is:%s\n", nodeIP, nodeID)
 	})
@@ -100,16 +100,12 @@ var _ = cluster.SIGDescribe("[Medium] [Serial] Drain node", func() {
 
 	ginkgo.It("drain node", func() {
 		ginkgo.By("drain node")
-		err := drainNode(nodeID)
+		err := drainAgentNode(nodeID)
 		framework.ExpectNoError(err)
 
 		ginkgo.By("wait for node not found")
-		err = fcluster.WaitForNodeNotFound(f.Client, nodeIP, f.Timeouts.CommonTimeout)
+		err = node.WaitForNodeNotFound(f.Client, nodeIP, f.Timeouts.CommonTimeout)
 		framework.ExpectNoError(err)
 		framework.Logf("node %s drained\n", nodeIP)
 	})
 })
-
-func drainNode(nodeID string) error {
-	return runCmd(fmt.Sprintf("kcctl drain --agent %s", nodeID))
-}
diff --git a/test/e2e/kcctl/kcctl.go b/test/e2e/kcctl/kcctl.go
index 28321a2..e077bbf 100644
--- a/test/e2e/kcctl/kcctl.go
+++ b/test/e2e/kcctl/kcctl.go
@@ -1,1 +1,96 @@
 package kcctl
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/onsi/ginkgo"
+
+	"github.com/kubeclipper/kubeclipper/pkg/query"
+	"github.com/kubeclipper/kubeclipper/pkg/scheme/common"
+	"github.com/kubeclipper/kubeclipper/pkg/simple/client/kc"
+	"github.com/kubeclipper/kubeclipper/pkg/utils/cmdutil"
+	"github.com/kubeclipper/kubeclipper/test/framework"
+	"github.com/kubeclipper/kubeclipper/test/framework/cluster"
+	"github.com/kubeclipper/kubeclipper/test/framework/node"
+)
+
+var _ = SIGDescribe("[Serial]", func() {
+	f := framework.NewDefaultFramework("kcctl")
+	var nodeID, nodeIP string
+
+	ginkgo.BeforeEach(func() {
+		ginkgo.By("Check that there are enough available nodes")
+		nodes, err := f.Client.ListNodes(context.TODO(), kc.Queries{
+			Pagination:    query.NoPagination(),
+			LabelSelector: fmt.Sprintf("!%s", common.LabelNodeDisable),
+		})
+		framework.ExpectNoError(err)
+		nodeID = nodes.Items[0].Name
+		nodeIP = nodes.Items[0].Status.Ipv4DefaultIP
+	})
+
+	//TODO: spoilt, need fix
+	ginkgo.It("[Medium] [Kcctl] [Join] [Drain] should join node to kubeclipper platform and drain it", func() {
+		ginkgo.By("drain node")
+		err := drainAgentNode(nodeID)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("wait for node not found")
+		err = node.WaitForNodeNotFound(f.Client, nodeIP, f.Timeouts.CommonTimeout)
+		framework.ExpectNoError(err)
+		framework.Logf("node %s drained\n", nodeIP)
+
+		ginkgo.By("drain node")
+		err = drainAgentNode(nodeID)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("wait for node not found")
+		err = node.WaitForNodeNotFound(f.Client, nodeIP, f.Timeouts.CommonTimeout)
+		framework.ExpectNoError(err)
+		framework.Logf("node %s drained\n", nodeIP)
+
+		ginkgo.By("join node")
+		err = joinAgentNode(nodeIP)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("wait for node join")
+		nodeID, err = node.WaitForNodeJoin(f.Client, nodeIP, f.Timeouts.CommonTimeout)
+		framework.ExpectNoError(err)
+		framework.Logf("node %s registered，id is:%s\n", nodeIP, nodeID)
+	})
+	ginkgo.It("[Medium] [Kcctl] [Drain] [Force] should drain node from kubeclipper platform force", func() {
+		ginkgo.By("drain agent node")
+		err := drainAgentNodeForce(nodeID)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("waiting for node is deleted")
+		err = node.WaitForNodeNotFound(f.Client, nodeID, f.Timeouts.CommonTimeout)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("join agent node")
+		err = joinAgentNode(nodeIP)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("wait for node join")
+		err = cluster.WaitForJoinNode(f.Client, nodeIP, f.Timeouts.CommonTimeout)
+		framework.ExpectNoError(err)
+	})
+})
+
+func joinAgentNode(nodeIP string) error {
+	return runCmd(fmt.Sprintf("kcctl join --agent %s", nodeIP))
+}
+
+func drainAgentNode(nodeID string) error {
+	return runCmd(fmt.Sprintf("kcctl drain --agent %s", nodeID))
+}
+
+func drainAgentNodeForce(nodeID string) error {
+	return runCmd(fmt.Sprintf("echo y | kcctl drain --agent %s -F", nodeID))
+}
+
+func runCmd(cmd string) error {
+	ec := cmdutil.NewExecCmd(context.TODO(), "bash", "-c", cmd)
+	return ec.Run()
+}
diff --git a/test/e2e/node/disable_node.go b/test/e2e/node/disable_node.go
deleted file mode 100644
index d29e08d..0000000
--- a/test/e2e/node/disable_node.go
+++ /dev/null
@@ -1,117 +0,0 @@
-/*
- *
- *  * Copyright 2021 KubeClipper Authors.
- *  *
- *  * Licensed under the Apache License, Version 2.0 (the "License");
- *  * you may not use this file except in compliance with the License.
- *  * You may obtain a copy of the License at
- *  *
- *  *     http://www.apache.org/licenses/LICENSE-2.0
- *  *
- *  * Unless required by applicable law or agreed to in writing, software
- *  * distributed under the License is distributed on an "AS IS" BASIS,
- *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- *  * See the License for the specific language governing permissions and
- *  * limitations under the License.
- *
- */
-
-package node
-
-import (
-	"context"
-	"fmt"
-
-	"github.com/kubeclipper/kubeclipper/pkg/query"
-	"github.com/kubeclipper/kubeclipper/pkg/simple/client/kc"
-	cluster2 "github.com/kubeclipper/kubeclipper/test/e2e/cluster"
-
-	"github.com/onsi/ginkgo"
-
-	"github.com/kubeclipper/kubeclipper/pkg/scheme/common"
-	"github.com/kubeclipper/kubeclipper/test/framework"
-)
-
-var _ = SIGDescribe("[Fast] [Serial] Disable node", func() {
-	f := framework.NewDefaultFramework("node")
-	nodeID := ""
-
-	f.AddAfterEach("enable the disable node", func(f *framework.Framework, failed bool) {
-		ginkgo.By("enable node")
-		err := f.Client.EnableNode(context.TODO(), nodeID)
-		framework.ExpectNoError(err)
-	})
-
-	ginkgo.BeforeEach(func() {
-		ginkgo.By("Check that there are enough available nodes")
-		nodes, err := f.Client.ListNodes(context.TODO(), kc.Queries{
-			Pagination:    query.NoPagination(),
-			LabelSelector: fmt.Sprintf("!%s", common.LabelNodeDisable),
-		})
-		framework.ExpectNoError(err)
-		if len(nodes.Items) == 0 {
-			framework.Failf("Not enough enabled nodes to test")
-		}
-		nodeID = nodes.Items[0].Name
-	})
-
-	ginkgo.It("disable a node and ensure the node is disabled", func() {
-		ginkgo.By("disable node")
-		err := f.Client.DisableNode(context.TODO(), nodeID)
-		framework.ExpectNoError(err)
-
-		ginkgo.By("check node is disabled")
-		nodeList, err := f.Client.DescribeNode(context.TODO(), nodeID)
-		framework.ExpectNoError(err)
-		if _, ok := nodeList.Items[0].Labels[common.LabelNodeDisable]; !ok {
-			framework.Failf("Fail to disable node")
-		} else {
-			ginkgo.By("node is disabled")
-		}
-	})
-})
-
-var _ = cluster2.SIGDescribe("[Fast] [Serial] Enable node", func() {
-	f := framework.NewDefaultFramework("node")
-	nodeID := ""
-
-	f.AddAfterEach("disable the enabled node", func(f *framework.Framework, failed bool) {
-		ginkgo.By("disable node")
-		err := f.Client.DisableNode(context.TODO(), nodeID)
-		framework.ExpectNoError(err)
-	})
-
-	ginkgo.BeforeEach(func() {
-		ginkgo.By("Check that there are enough available nodes")
-		nodes, err := f.Client.ListNodes(context.TODO(), kc.Queries{
-			Pagination:    query.NoPagination(),
-			LabelSelector: common.LabelNodeDisable,
-		})
-		framework.ExpectNoError(err)
-		if len(nodes.Items) == 0 {
-			framework.Failf("Not enough disabled nodes to test")
-		}
-		nodeID = nodes.Items[0].Name
-
-		if _, ok := nodes.Items[0].Labels[common.LabelNodeDisable]; !ok {
-			ginkgo.By("Make sure the node is disabled")
-			err := f.Client.DisableNode(context.TODO(), nodeID)
-			framework.ExpectNoError(err)
-		}
-	})
-
-	ginkgo.It("enable a node and ensure the node is enabled", func() {
-		ginkgo.By("enable node")
-		err := f.Client.EnableNode(context.TODO(), nodeID)
-		framework.ExpectNoError(err)
-
-		ginkgo.By("check node is enabled")
-		nodeList, err := f.Client.DescribeNode(context.TODO(), nodeID)
-		framework.ExpectNoError(err)
-		if _, ok := nodeList.Items[0].Labels[common.LabelNodeDisable]; !ok {
-			ginkgo.By("node is enabled")
-		} else {
-			framework.Failf("Fail to enabled node")
-		}
-	})
-})
diff --git a/test/e2e/node/spec.go b/test/e2e/node/spec.go
new file mode 100644
index 0000000..7905f8c
--- /dev/null
+++ b/test/e2e/node/spec.go
@@ -0,0 +1,121 @@
+package node
+
+import (
+	"context"
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	"github.com/gorilla/websocket"
+	"github.com/onsi/ginkgo"
+
+	v1 "github.com/kubeclipper/kubeclipper/pkg/apis/core/v1"
+	"github.com/kubeclipper/kubeclipper/pkg/query"
+	"github.com/kubeclipper/kubeclipper/pkg/scheme/common"
+	"github.com/kubeclipper/kubeclipper/pkg/simple/client/kc"
+	"github.com/kubeclipper/kubeclipper/pkg/utils/certs"
+	"github.com/kubeclipper/kubeclipper/test/framework"
+)
+
+var _ = SIGDescribe("[Serial]", func() {
+	f := framework.NewDefaultFramework("node")
+	var nodeID, nodeIP string
+
+	ginkgo.BeforeEach(func() {
+		ginkgo.By("Check that there are enough available nodes")
+		nodes, err := f.Client.ListNodes(context.TODO(), kc.Queries{
+			Pagination:    query.NoPagination(),
+			LabelSelector: fmt.Sprintf("!%s", common.LabelNodeDisable),
+		})
+		framework.ExpectNoError(err)
+		nodeID = nodes.Items[0].Name
+		nodeIP = nodes.Items[0].Status.Ipv4DefaultIP
+	})
+
+	ginkgo.It("[Fast] [new-node] [Enable] [Disable] should disable and enable node of kubeclipper platform", func() {
+		ginkgo.By("disable node")
+		err := f.Client.DisableNode(context.TODO(), nodeID)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("check node is disabled")
+		nodeList, err := f.Client.DescribeNode(context.TODO(), nodeID)
+		framework.ExpectNoError(err)
+		if _, ok := nodeList.Items[0].Labels[common.LabelNodeDisable]; !ok {
+			framework.Failf("Fail to disable node")
+		} else {
+			ginkgo.By("node is disabled")
+		}
+
+		ginkgo.By("enable node")
+		err = f.Client.EnableNode(context.TODO(), nodeID)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("check node is enabled")
+		nodeList, err = f.Client.DescribeNode(context.TODO(), nodeID)
+		framework.ExpectNoError(err)
+		if _, ok := nodeList.Items[0].Labels[common.LabelNodeDisable]; !ok {
+			ginkgo.By("node is enabled")
+		} else {
+			framework.Failf("Fail to enabled node")
+		}
+	})
+	ginkgo.It("[Fast] [new-node] [Terminal] should connect node ssh", func() {
+		ginkgo.By("Get public key")
+		pub, err := f.Client.GetPublicKey(context.TODO())
+		framework.ExpectNoError(err)
+
+		ginkgo.By("Get msg")
+		pubkey, _ := base64.StdEncoding.DecodeString(pub.PublicKey)
+		u, err := certs.RsaEncrypt([]byte("root"), pubkey)
+		framework.ExpectNoError(err)
+		p, err := certs.RsaEncrypt([]byte("Thinkbig1"), pubkey)
+		framework.ExpectNoError(err)
+		credential := &v1.SSHCredential{
+			Username: u,
+			Password: p,
+			Port:     22,
+		}
+		jsonBody, err := json.Marshal(credential)
+		msg := base64.StdEncoding.EncodeToString(jsonBody)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("connect node")
+		url := fmt.Sprintf("ws://%s%s/%s/%sname=%s&token=%s&msg=%s", f.Client.Host(), kc.ListNodesPath, nodeID, "terminal?", nodeID, f.Client.Token(), msg)
+		ws, _, err := websocket.DefaultDialer.Dial(url, nil)
+		framework.ExpectNoError(err)
+
+		ginkgo.By("check the node is connected")
+		for {
+			_, data, err := ws.ReadMessage()
+			if string(data) == "" || err != nil {
+				framework.Failf("Fail to connect node")
+			} else {
+				break
+			}
+		}
+
+		ginkgo.By("close the connection")
+		err = ws.Close()
+		framework.ExpectNoError(err)
+	})
+	ginkgo.It("[Fast] [new-node] [Detail] [Info] should get info and detail of node", func() {
+		ginkgo.By("show node info")
+		nodeList, err := f.Client.ListNodes(context.TODO(), kc.Queries{
+			Pagination: query.NoPagination(),
+			FuzzySearch: map[string]string{
+				"default-ip": nodeIP,
+			},
+		})
+		framework.ExpectNoError(err)
+		if len(nodeList.Items) == 0 {
+			framework.Failf("show node info e2e test failed, no such node")
+		}
+
+		ginkgo.By("show node detail")
+		nodeList, err = f.Client.DescribeNode(context.TODO(), nodeID)
+		framework.ExpectNoError(err)
+		if len(nodeList.Items) == 0 {
+			framework.Failf("show node detail e2e test failed, no such node")
+		}
+	})
+})
diff --git a/test/e2e/node/terminal.go b/test/e2e/node/terminal.go
deleted file mode 100644
index 0df7c4d..0000000
--- a/test/e2e/node/terminal.go
+++ /dev/null
@@ -1,74 +0,0 @@
-package node
-
-import (
-	"context"
-	"encoding/base64"
-	"encoding/json"
-	"fmt"
-
-	"github.com/gorilla/websocket"
-	"github.com/onsi/ginkgo"
-
-	v1 "github.com/kubeclipper/kubeclipper/pkg/apis/core/v1"
-	"github.com/kubeclipper/kubeclipper/pkg/query"
-	"github.com/kubeclipper/kubeclipper/pkg/simple/client/kc"
-	"github.com/kubeclipper/kubeclipper/pkg/utils/certs"
-	"github.com/kubeclipper/kubeclipper/test/framework"
-)
-
-var _ = SIGDescribe("[Fast] [Serial] Node terminal connect", func() {
-	f := framework.NewDefaultFramework("node")
-	msg := ""
-	nodeID := ""
-	credential := &v1.SSHCredential{}
-
-	ginkgo.BeforeEach(func() {
-		ginkgo.By("Check that there are enough available nodes")
-		nodes, err := f.Client.ListNodes(context.TODO(), kc.Queries{
-			Pagination: query.NoPagination(),
-		})
-		framework.ExpectNoError(err)
-		nodeID = nodes.Items[0].Name
-
-		ginkgo.By("Get public key")
-		pub, err := f.Client.GetPublicKey(context.TODO())
-		framework.ExpectNoError(err)
-
-		ginkgo.By("Get msg")
-		pubkey, _ := base64.StdEncoding.DecodeString(pub.PublicKey)
-		u, err := certs.RsaEncrypt([]byte("root"), pubkey)
-		framework.ExpectNoError(err)
-		p, err := certs.RsaEncrypt([]byte("Thinkbig1"), pubkey)
-		framework.ExpectNoError(err)
-
-		credential = &v1.SSHCredential{
-			Username: u,
-			Password: p,
-			Port:     22,
-		}
-		jsonBody, err := json.Marshal(credential)
-		msg = base64.StdEncoding.EncodeToString(jsonBody)
-		framework.ExpectNoError(err)
-	})
-
-	ginkgo.It("connect the node and ensure node is connected", func() {
-		ginkgo.By("connect node")
-		url := fmt.Sprintf("ws://%s%s/%s/%sname=%s&token=%s&msg=%s", f.Client.Host(), kc.ListNodesPath, nodeID, "terminal?", nodeID, f.Client.Token(), msg)
-		ws, _, err := websocket.DefaultDialer.Dial(url, nil)
-		framework.ExpectNoError(err)
-
-		ginkgo.By("check the node is connected")
-		for {
-			_, data, err := ws.ReadMessage()
-			if string(data) == "" || err != nil {
-				framework.Failf("Fail to connect node")
-			} else {
-				break
-			}
-		}
-
-		ginkgo.By("close the connection")
-		err = ws.Close()
-		framework.ExpectNoError(err)
-	})
-})
diff --git a/test/e2e/region/framework.go b/test/e2e/region/framework.go
new file mode 100644
index 0000000..a224c57
--- /dev/null
+++ b/test/e2e/region/framework.go
@@ -0,0 +1,26 @@
+/*
+ *
+ *  * Copyright 2021 KubeClipper Authors.
+ *  *
+ *  * Licensed under the Apache License, Version 2.0 (the "License");
+ *  * you may not use this file except in compliance with the License.
+ *  * You may obtain a copy of the License at
+ *  *
+ *  *     http://www.apache.org/licenses/LICENSE-2.0
+ *  *
+ *  * Unless required by applicable law or agreed to in writing, software
+ *  * distributed under the License is distributed on an "AS IS" BASIS,
+ *  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  * See the License for the specific language governing permissions and
+ *  * limitations under the License.
+ *
+ */
+
+package region
+
+import "github.com/onsi/ginkgo"
+
+// SIGDescribe annotates the test with the SIG label.
+func SIGDescribe(text string, body func()) bool {
+	return ginkgo.Describe("[sig-region] "+text, body)
+}
diff --git a/test/e2e/region/region.go b/test/e2e/region/region.go
new file mode 100644
index 0000000..aabc6d0
--- /dev/null
+++ b/test/e2e/region/region.go
@@ -0,0 +1,60 @@
+package region
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/onsi/ginkgo"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/kubeclipper/kubeclipper/pkg/query"
+	v1 "github.com/kubeclipper/kubeclipper/pkg/scheme/core/v1"
+	"github.com/kubeclipper/kubeclipper/pkg/simple/client/kc"
+	"github.com/kubeclipper/kubeclipper/test/framework"
+)
+
+var _ = SIGDescribe("[Fast] [Serial] List Region", func() {
+	f := framework.NewDefaultFramework("region")
+	ginkgo.It("list region and check is default region exist", func() {
+		ctx := context.Background()
+		ginkgo.By("list region")
+		q := query.New()
+		q.Limit = -1
+		list, err := f.Client.ListRegion(ctx, kc.Queries(*q))
+		framework.ExpectNoError(err)
+
+		if getDefault(list.Items) == nil {
+			ginkgo.Fail("default region not exist")
+		}
+
+		ginkgo.By("search default region")
+		q = query.New()
+		q.Limit = -1
+		q.FuzzySearch = map[string]string{"name": "default"}
+		list, err = f.Client.ListRegion(ctx, kc.Queries(*q))
+		framework.ExpectNoError(err)
+		region := getDefault(list.Items)
+		if region == nil {
+			ginkgo.Fail("default region not exist")
+		}
+		if !isValidTime(region.CreationTimestamp) {
+			ginkgo.Fail(fmt.Sprintf("default region's create time(%s) is invalid", region.CreationTimestamp.Format(time.RFC3339)))
+		}
+	})
+})
+
+func getDefault(regions []v1.Region) *v1.Region {
+	for _, region := range regions {
+		if region.Name == "default" {
+			return &region
+		}
+	}
+	return nil
+}
+
+// isValidTime if 0 < time.unix < now, we think it's a valid time
+func isValidTime(t metav1.Time) bool {
+	now := metav1.Now()
+	return t.Before(&now) && t.After(time.Unix(0, 0))
+}
diff --git a/test/framework/cluster/wait.go b/test/framework/cluster/wait.go
index f5c2814..c1225a8 100644
--- a/test/framework/cluster/wait.go
+++ b/test/framework/cluster/wait.go
@@ -15,37 +15,6 @@ import (
 	"github.com/kubeclipper/kubeclipper/test/framework"
 )
 
-const (
-	// poll is how often to poll clusters.
-	poll = 15 * time.Second
-)
-
-type timeoutError struct {
-	msg             string
-	observedObjects []interface{}
-}
-
-func (e *timeoutError) Error() string {
-	return e.msg
-}
-
-func TimeoutError(msg string, observedObjects ...interface{}) error {
-	return &timeoutError{
-		msg:             msg,
-		observedObjects: observedObjects,
-	}
-}
-
-func IsTimeout(err error) bool {
-	if err == wait.ErrWaitTimeout {
-		return true
-	}
-	if _, ok := err.(*timeoutError); ok {
-		return true
-	}
-	return false
-}
-
 type clusterCondition func(clu *corev1.Cluster) (bool, error)
 
 type backupCondition func(backup *corev1.Backup) (bool, error)
@@ -58,11 +27,11 @@ func WaitForClusterCondition(c *kc.Client, clusterName, conditionDesc string, ti
 		lastCluster      *corev1.Cluster
 		start            = time.Now()
 	)
-	err := wait.PollImmediate(poll, timeout, func() (bool, error) {
+	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
 		clu, err := c.DescribeCluster(context.TODO(), clusterName)
 		lastClusterError = err
 		if err != nil || len(clu.Items) == 0 {
-			return handleWaitingAPIError(err, true, "getting cluster %s", clusterName)
+			return framework.HandleWaitingAPIError(err, true, "getting cluster %s", clusterName)
 		}
 		lastCluster = clu.Items[0].DeepCopy()
 		framework.Logf("Cluster %q: Phase=%q, Elapsed: %v",
@@ -81,8 +50,8 @@ func WaitForClusterCondition(c *kc.Client, clusterName, conditionDesc string, ti
 	if err == nil {
 		return nil
 	}
-	if IsTimeout(err) && lastCluster != nil {
-		return TimeoutError(fmt.Sprintf("timed out while waiting for cluster %s to be %s", clusterName, conditionDesc),
+	if framework.IsTimeout(err) && lastCluster != nil {
+		return framework.TimeoutError(fmt.Sprintf("timed out while waiting for cluster %s to be %s", clusterName, conditionDesc),
 			lastCluster,
 		)
 	}
@@ -90,17 +59,17 @@ func WaitForClusterCondition(c *kc.Client, clusterName, conditionDesc string, ti
 		// If the last API call was an error.
 		err = lastClusterError
 	}
-	return maybeTimeoutError(err, "waiting for cluster %s to be %s", clusterName, conditionDesc)
+	return framework.MaybeTimeoutError(err, "waiting for cluster %s to be %s", clusterName, conditionDesc)
 }
 
 func WaitForBackupCondition(c *kc.Client, clusterName, backupName, conditionDesc string, timeout time.Duration, condition backupCondition) error {
 	framework.Logf("Waiting up to %v for backup %q to be %q", timeout, backupName, conditionDesc)
 	bp := &corev1.Backup{}
 	start := time.Now()
-	err := wait.PollImmediate(poll, timeout, func() (bool, error) {
+	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
 		backups, apiErr := c.ListBackupsWithCluster(context.TODO(), clusterName)
 		if apiErr != nil || len(backups.Items) == 0 {
-			return handleWaitingAPIError(apiErr, true, "getting backup %s", backupName)
+			return framework.HandleWaitingAPIError(apiErr, true, "getting backup %s", backupName)
 		}
 		bp = backups.Items[0].DeepCopy()
 		framework.Logf("Backup %q: Phase=%q, Elapsed: %v", backupName, bp.Status.ClusterBackupStatus, time.Since(start))
@@ -117,10 +86,10 @@ func WaitForBackupCondition(c *kc.Client, clusterName, backupName, conditionDesc
 	if err == nil {
 		return nil
 	}
-	if IsTimeout(err) && bp != nil {
-		return TimeoutError(fmt.Sprintf("timed out while waiting for backup %s to be %s", backupName, conditionDesc), bp)
+	if framework.IsTimeout(err) && bp != nil {
+		return framework.TimeoutError(fmt.Sprintf("timed out while waiting for backup %s to be %s", backupName, conditionDesc), bp)
 	}
-	return maybeTimeoutError(err, "waiting for backup %s to be %s", backupName, conditionDesc)
+	return framework.MaybeTimeoutError(err, "waiting for backup %s to be %s", backupName, conditionDesc)
 }
 
 func WaitForClusterRunning(c *kc.Client, clusterName string, timeout time.Duration) error {
@@ -146,14 +115,14 @@ func WaitForClusterHealthy(c *kc.Client, clusterName string, timeout time.Durati
 // than "not found" then that error is returned and the wait stops.
 func WaitForClusterNotFound(c *kc.Client, clusterName string, timeout time.Duration) error {
 	var lastCluster *corev1.Cluster
-	err := wait.PollImmediate(poll, timeout, func() (done bool, err error) {
+	err := wait.PollImmediate(framework.Poll, timeout, func() (done bool, err error) {
 		clu, err := c.DescribeCluster(context.TODO(), clusterName)
 		if apierror.IsNotFound(err) {
 			// done
 			return true, nil
 		}
 		if err != nil {
-			return handleWaitingAPIError(err, true, "getting cluster %s", clusterName)
+			return framework.HandleWaitingAPIError(err, true, "getting cluster %s", clusterName)
 		}
 		if len(clu.Items) == 0 {
 			framework.Logf("unexpected problem, cluster not be nil at this time")
@@ -165,18 +134,18 @@ func WaitForClusterNotFound(c *kc.Client, clusterName string, timeout time.Durat
 	if err == nil {
 		return nil
 	}
-	if IsTimeout(err) && lastCluster != nil {
-		return TimeoutError(fmt.Sprintf("timeout while waiting for cluster %s to be Not Found", clusterName), lastCluster)
+	if framework.IsTimeout(err) && lastCluster != nil {
+		return framework.TimeoutError(fmt.Sprintf("timeout while waiting for cluster %s to be Not Found", clusterName), lastCluster)
 	}
-	return maybeTimeoutError(err, "waiting for cluster %s not found", clusterName)
+	return framework.MaybeTimeoutError(err, "waiting for cluster %s not found", clusterName)
 }
 
 func WaitForComponentNotFound(c *kc.Client, clusterName string, timeout time.Duration) error {
 	var lastCluster *corev1.Cluster
-	err := wait.PollImmediate(poll, timeout, func() (done bool, err error) {
+	err := wait.PollImmediate(framework.Poll, timeout, func() (done bool, err error) {
 		clu, err := c.DescribeCluster(context.TODO(), clusterName)
 		if err != nil {
-			return handleWaitingAPIError(err, true, "getting cluster %s", clusterName)
+			return framework.HandleWaitingAPIError(err, true, "getting cluster %s", clusterName)
 		}
 		if len(clu.Items) == 0 {
 			framework.Logf("unexpected problem, cluster not be nil at this time")
@@ -191,10 +160,10 @@ func WaitForComponentNotFound(c *kc.Client, clusterName string, timeout time.Dur
 	if err == nil {
 		return nil
 	}
-	if IsTimeout(err) && lastCluster != nil {
-		return TimeoutError(fmt.Sprintf("timeout while waiting for cluster %s component to be Not Found", clusterName), lastCluster)
+	if framework.IsTimeout(err) && lastCluster != nil {
+		return framework.TimeoutError(fmt.Sprintf("timeout while waiting for cluster %s component to be Not Found", clusterName), lastCluster)
 	}
-	return maybeTimeoutError(err, "waiting for cluster %s uninstall component not found", clusterName)
+	return framework.MaybeTimeoutError(err, "waiting for cluster %s uninstall component not found", clusterName)
 }
 
 func WaitForBackupAvailable(c *kc.Client, clusterName, backupName string, timeout time.Duration) error {
@@ -211,10 +180,10 @@ func WaitForBackupAvailable(c *kc.Client, clusterName, backupName string, timeou
 
 func WaitForBackupNotFound(c *kc.Client, clusterName, backupName string, timeout time.Duration) error {
 	bp := &corev1.Backup{}
-	err := wait.PollImmediate(poll, timeout, func() (done bool, err error) {
+	err := wait.PollImmediate(framework.Poll, timeout, func() (done bool, err error) {
 		backups, waitErr := c.ListBackupsWithCluster(context.TODO(), clusterName)
 		if waitErr != nil {
-			return handleWaitingAPIError(waitErr, true, "getting backup %s", backupName)
+			return framework.HandleWaitingAPIError(waitErr, true, "getting backup %s", backupName)
 		}
 		if len(backups.Items) == 0 {
 			return true, nil
@@ -225,10 +194,10 @@ func WaitForBackupNotFound(c *kc.Client, clusterName, backupName string, timeout
 	if err == nil {
 		return nil
 	}
-	if IsTimeout(err) && bp != nil {
-		return TimeoutError(fmt.Sprintf("timeout while waiting for backup %s to be Not Found", backupName), bp)
+	if framework.IsTimeout(err) && bp != nil {
+		return framework.TimeoutError(fmt.Sprintf("timeout while waiting for backup %s to be Not Found", backupName), bp)
 	}
-	return maybeTimeoutError(err, "waiting for backup %s not found", backupName)
+	return framework.MaybeTimeoutError(err, "waiting for backup %s not found", backupName)
 }
 
 func WaitForRecovery(c *kc.Client, clusterName string, timeout time.Duration) error {
@@ -305,10 +274,10 @@ func WaitForJoinNode(c *kc.Client, nodeIP string, timeout time.Duration) error {
 	query := kc.Queries{
 		FieldSelector: fmt.Sprintf("status.ipv4DefaultIP=%s", nodeIP),
 	}
-	err := wait.PollImmediate(poll, timeout, func() (done bool, err error) {
+	err := wait.PollImmediate(framework.Poll, timeout, func() (done bool, err error) {
 		nodes, waitErr := c.ListNodes(context.TODO(), query)
 		if waitErr != nil {
-			return handleWaitingAPIError(waitErr, true, "getting node %s", nodeIP)
+			return framework.HandleWaitingAPIError(waitErr, true, "getting node %s", nodeIP)
 		}
 		if len(nodes.Items) == 0 {
 			return true, nil
@@ -319,51 +288,8 @@ func WaitForJoinNode(c *kc.Client, nodeIP string, timeout time.Duration) error {
 	if err == nil {
 		return nil
 	}
-	if IsTimeout(err) && node != nil {
-		return TimeoutError(fmt.Sprintf("timeout while waiting for node %s to be join", nodeIP), node)
-	}
-	return maybeTimeoutError(err, "waiting for node %s join", nodeIP)
-}
-
-// maybeTimeoutError returns a TimeoutError if err is a timeout. Otherwise, wrap err.
-// taskFormat and taskArgs should be the task being performed when the error occurred,
-// e.g. "waiting for pod to be running".
-func maybeTimeoutError(err error, taskFormat string, taskArgs ...interface{}) error {
-	if IsTimeout(err) {
-		return TimeoutError(fmt.Sprintf("timed out while "+taskFormat, taskArgs...))
-	} else if err != nil {
-		return fmt.Errorf("error while %s: %w", fmt.Sprintf(taskFormat, taskArgs...), err)
-	} else {
-		return nil
-	}
-}
-
-// handleWaitingAPIErrror handles an error from an API request in the context of a Wait function.
-// If the error is retryable, sleep the recommended delay and ignore the error.
-// If the erorr is terminal, return it.
-func handleWaitingAPIError(err error, retryNotFound bool, taskFormat string, taskArgs ...interface{}) (bool, error) {
-	taskDescription := fmt.Sprintf(taskFormat, taskArgs...)
-	if retryNotFound && apierror.IsNotFound(err) {
-		framework.Logf("Ignoring NotFound error while " + taskDescription)
-		return false, nil
-	}
-	if retry, delay := shouldRetry(err); retry {
-		framework.Logf("Retryable error while %s, retrying after %v: %v", taskDescription, delay, err)
-		if delay > 0 {
-			time.Sleep(delay)
-		}
-		return false, nil
+	if framework.IsTimeout(err) && node != nil {
+		return framework.TimeoutError(fmt.Sprintf("timeout while waiting for node %s to be join", nodeIP), node)
 	}
-	framework.Logf("Encountered non-retryable error while %s: %v", taskDescription, err)
-	return false, err
-}
-
-// Decide whether to retry an API request. Optionally include a delay to retry after.
-func shouldRetry(err error) (retry bool, retryAfter time.Duration) {
-	// these errors indicate a transient error that should be retried.
-	if apierror.IsInternalError(err) || apierror.IsTooManyRequests(err) {
-		return true, 0
-	}
-
-	return false, 0
+	return framework.MaybeTimeoutError(err, "waiting for node %s join", nodeIP)
 }
diff --git a/test/framework/cluster/node.go b/test/framework/node/node.go
similarity index 79%
rename from test/framework/cluster/node.go
rename to test/framework/node/node.go
index 854ed3b..b1f18b5 100644
--- a/test/framework/cluster/node.go
+++ b/test/framework/node/node.go
@@ -1,4 +1,4 @@
-package cluster
+package node
 
 import (
 	"context"
@@ -18,10 +18,10 @@ type nodeCondition func(nodes []corev1.Node) (bool, error)
 func waitForNodeCondition(c *kc.Client, nodeIP, conditionDesc string, timeout time.Duration, condition nodeCondition) error {
 	framework.Logf("Waiting up to %v for node %q to be %q", timeout, nodeIP, conditionDesc)
 	start := time.Now()
-	err := wait.PollImmediate(poll, timeout, func() (bool, error) {
+	err := wait.PollImmediate(framework.Poll, timeout, func() (bool, error) {
 		nodes, apiErr := c.ListNodes(context.TODO(), kc.Queries{})
 		if apiErr != nil {
-			return handleWaitingAPIError(apiErr, true, "getting node %s", nodeIP)
+			return framework.HandleWaitingAPIError(apiErr, true, "getting node %s", nodeIP)
 		}
 		ok, err := condition(nodes.Items)
 		if err != nil {
@@ -33,10 +33,10 @@ func waitForNodeCondition(c *kc.Client, nodeIP, conditionDesc string, timeout ti
 	if err == nil {
 		return nil
 	}
-	if IsTimeout(err) {
-		return TimeoutError(fmt.Sprintf("timed out while waiting for node %s to be %s", nodeIP, conditionDesc))
+	if framework.IsTimeout(err) {
+		return framework.TimeoutError(fmt.Sprintf("timed out while waiting for node %s to be %s", nodeIP, conditionDesc))
 	}
-	return maybeTimeoutError(err, "waiting for node %s to be %s", nodeIP, conditionDesc)
+	return framework.MaybeTimeoutError(err, "waiting for node %s to be %s", nodeIP, conditionDesc)
 }
 
 func WaitForNodeJoin(c *kc.Client, nodeIP string, timeout time.Duration) (string, error) {
diff --git a/test/framework/wait.go b/test/framework/wait.go
new file mode 100644
index 0000000..926705f
--- /dev/null
+++ b/test/framework/wait.go
@@ -0,0 +1,84 @@
+package framework
+
+import (
+	"fmt"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	apierror "github.com/kubeclipper/kubeclipper/pkg/errors"
+)
+
+const (
+	// Poll is how often to poll clusters.
+	Poll = 15 * time.Second
+)
+
+type timeoutError struct {
+	msg             string
+	observedObjects []interface{}
+}
+
+func (e *timeoutError) Error() string {
+	return e.msg
+}
+
+func TimeoutError(msg string, observedObjects ...interface{}) error {
+	return &timeoutError{
+		msg:             msg,
+		observedObjects: observedObjects,
+	}
+}
+
+func IsTimeout(err error) bool {
+	if err == wait.ErrWaitTimeout {
+		return true
+	}
+	if _, ok := err.(*timeoutError); ok {
+		return true
+	}
+	return false
+}
+
+// MaybeTimeoutError returns a TimeoutError if err is a timeout. Otherwise, wrap err.
+// taskFormat and taskArgs should be the task being performed when the error occurred,
+// e.g. "waiting for pod to be running".
+func MaybeTimeoutError(err error, taskFormat string, taskArgs ...interface{}) error {
+	if IsTimeout(err) {
+		return TimeoutError(fmt.Sprintf("timed out while "+taskFormat, taskArgs...))
+	} else if err != nil {
+		return fmt.Errorf("error while %s: %w", fmt.Sprintf(taskFormat, taskArgs...), err)
+	} else {
+		return nil
+	}
+}
+
+// HandleWaitingAPIError handles an error from an API request in the context of a Wait function.
+// If the error is retryable, sleep the recommended delay and ignore the error.
+// If the erorr is terminal, return it.
+func HandleWaitingAPIError(err error, retryNotFound bool, taskFormat string, taskArgs ...interface{}) (bool, error) {
+	taskDescription := fmt.Sprintf(taskFormat, taskArgs...)
+	if retryNotFound && apierror.IsNotFound(err) {
+		Logf("Ignoring NotFound error while " + taskDescription)
+		return false, nil
+	}
+	if retry, delay := shouldRetry(err); retry {
+		Logf("Retryable error while %s, retrying after %v: %v", taskDescription, delay, err)
+		if delay > 0 {
+			time.Sleep(delay)
+		}
+		return false, nil
+	}
+	Logf("Encountered non-retryable error while %s: %v", taskDescription, err)
+	return false, err
+}
+
+// Decide whether to retry an API request. Optionally include a delay to retry after.
+func shouldRetry(err error) (retry bool, retryAfter time.Duration) {
+	// these errors indicate a transient error that should be retried.
+	if apierror.IsInternalError(err) || apierror.IsTooManyRequests(err) {
+		return true, 0
+	}
+
+	return false, 0
+}