Repository: openyurtio/openyurt
Pull request #252 (2021-04-07 09:21:08)

refactor: move cmd.go from pkg dir to cmd dir for yurttunnel
<!-- 
Please make sure you have read and understood the contributing guidelines;
https://github.com/openyurtio/openyurt/blob/master/CONTRIBUTING.md -->

### Ⅰ. Describe what this PR does
In order to improve the code readability of yurttunnel, move `cmd.go` of yurttunnel out of `pkg` dir, and refactor `cmd.go` into dir: `options`, `config` and `start.go` 

### Ⅱ. Does this pull request fix one issue?
<!--If so, add "fixes #xxxx" below in the next line, for example, fixes #15. Otherwise, add "NONE" -->


### Ⅲ. List the added test cases (unit test/integration test) if any, please explain if no tests are needed.


### Ⅳ. Describe how to verify it
make all
make release WHAT=cmd/yurt-tunnel-server
make release WHAT=cmd/yurt-tunnel-agent
and make sure yurt-tunnel-server/yurt-tunnel-agent can work in openyurt cluster.

### Ⅴ. Special notes for reviews




charleszheng44: /lgtm

Patch:
diff --git a/cmd/yurt-tunnel-agent/agent.go b/cmd/yurt-tunnel-agent/agent.go
index d9a2915b..d8a29a19 100644
--- a/cmd/yurt-tunnel-agent/agent.go
+++ b/cmd/yurt-tunnel-agent/agent.go
@@ -19,18 +19,18 @@ package main
 import (
 	"flag"
 
+	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-agent/app"
+	"github.com/openyurtio/openyurt/pkg/projectinfo"
+
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
-
-	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/agent"
 )
 
 func main() {
 	klog.InitFlags(nil)
 	defer klog.Flush()
 
-	cmd := agent.NewYurttunnelAgentCommand(wait.NeverStop)
+	cmd := app.NewYurttunnelAgentCommand(wait.NeverStop)
 	cmd.Flags().AddGoFlagSet(flag.CommandLine)
 	if err := cmd.Execute(); err != nil {
 		klog.Fatalf("%s failed: %s", projectinfo.GetAgentName(), err)
diff --git a/cmd/yurt-tunnel-agent/app/config/config.go b/cmd/yurt-tunnel-agent/app/config/config.go
new file mode 100644
index 00000000..56b119ce
--- /dev/null
+++ b/cmd/yurt-tunnel-agent/app/config/config.go
@@ -0,0 +1,45 @@
+/*
+Copyright 2021 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import "k8s.io/client-go/kubernetes"
+
+// Config is the main context object for yurttunel-agent
+type Config struct {
+	NodeName         string
+	NodeIP           string
+	TunnelServerAddr string
+	Client           kubernetes.Interface
+	AgentIdentifiers string
+}
+
+type completedConfig struct {
+	*Config
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() *CompletedConfig {
+	cc := completedConfig{c}
+
+	return &CompletedConfig{&cc}
+}
diff --git a/cmd/yurt-tunnel-agent/app/options/options.go b/cmd/yurt-tunnel-agent/app/options/options.go
new file mode 100644
index 00000000..a662c5c2
--- /dev/null
+++ b/cmd/yurt-tunnel-agent/app/options/options.go
@@ -0,0 +1,143 @@
+/*
+Copyright 2021 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-agent/app/config"
+	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes"
+
+	"github.com/spf13/pflag"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/apiserver-network-proxy/pkg/agent"
+)
+
+const defaultKubeconfig = "/etc/kubernetes/kubelet.conf"
+
+// AgentOptions has the information that required by the yurttunel-agent
+type AgentOptions struct {
+	NodeName         string
+	NodeIP           string
+	TunnelServerAddr string
+	ApiserverAddr    string
+	KubeConfig       string
+	Version          bool
+	AgentIdentifiers string
+}
+
+// NewAgentOptions creates a new AgentOptions with a default config.
+func NewAgentOptions() *AgentOptions {
+	o := &AgentOptions{}
+
+	return o
+}
+
+// validate validates the AgentOptions
+func (o *AgentOptions) Validate() error {
+	if o.NodeName == "" {
+		o.NodeName = os.Getenv("NODE_NAME")
+		if o.NodeName == "" {
+			return errors.New("either --node-name or $NODE_NAME has to be set")
+		}
+	}
+
+	if o.NodeIP == "" {
+		o.NodeIP = os.Getenv("NODE_IP")
+		if o.NodeIP == "" {
+			return errors.New("either --node-ip or $NODE_IP has to be set")
+		}
+	}
+
+	if !agentIdentifiersAreValid(o.AgentIdentifiers) {
+		return errors.New("--agent-identifiers are invalid, format should be host={node-name}")
+	}
+
+	return nil
+}
+
+// AddFlags returns flags for a specific yurttunnel-agent by section name
+func (o *AgentOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.BoolVar(&o.Version, "version", o.Version, "print the version information.")
+	fs.StringVar(&o.NodeName, "node-name", o.NodeName, "The name of the edge node.")
+	fs.StringVar(&o.NodeIP, "node-ip", o.NodeIP, "The host IP of the edge node.")
+	fs.StringVar(&o.TunnelServerAddr, "tunnelserver-addr", o.TunnelServerAddr, fmt.Sprintf("The address of %s", projectinfo.GetServerName()))
+	fs.StringVar(&o.ApiserverAddr, "apiserver-addr", o.ApiserverAddr, "A reachable address of the apiserver.")
+	fs.StringVar(&o.KubeConfig, "kube-config", o.KubeConfig, "Path to the kubeconfig file.")
+	fs.StringVar(&o.AgentIdentifiers, "agent-identifiers", o.AgentIdentifiers, "The identifiers of the agent, which will be used by the server when choosing agent.")
+}
+
+// agentIdentifiersIsValid verify agent identifiers are valid or not.
+// and agentIdentifiers can be empty because default value will be set in complete() func.
+func agentIdentifiersAreValid(agentIdentifiers string) bool {
+	if len(agentIdentifiers) == 0 {
+		return true
+	}
+
+	entries := strings.Split(agentIdentifiers, ",")
+	for i := range entries {
+		parts := strings.Split(entries[i], "=")
+		if len(parts) != 2 {
+			return false
+		}
+
+		switch agent.IdentifierType(parts[0]) {
+		case agent.Host, agent.CIDR, agent.IPv4, agent.IPv6, agent.UID:
+			// valid agent identifier
+		default:
+			return false
+		}
+	}
+
+	return true
+}
+
+// Config return a yurttunnel agent config objective
+func (o *AgentOptions) Config() (*config.Config, error) {
+	var err error
+	c := &config.Config{
+		NodeName:         o.NodeName,
+		NodeIP:           o.NodeIP,
+		TunnelServerAddr: o.TunnelServerAddr,
+		AgentIdentifiers: o.AgentIdentifiers,
+	}
+
+	if len(c.AgentIdentifiers) == 0 {
+		c.AgentIdentifiers = fmt.Sprintf("ipv4=%s&host=%s", o.NodeIP, o.NodeName)
+	}
+	klog.Infof("%s is set for agent identifies", c.AgentIdentifiers)
+
+	kubeConfig := o.KubeConfig
+	if o.KubeConfig == "" && o.ApiserverAddr == "" {
+		kubeConfig = defaultKubeconfig
+		klog.Infof("neither --kube-config nor --apiserver-addr is set, will use %s as the kubeconfig", kubeConfig)
+	}
+
+	if kubeConfig != "" {
+		klog.Infof("create the clientset based on the kubeconfig(%s).", kubeConfig)
+		c.Client, err = kubeutil.CreateClientSetKubeConfig(kubeConfig)
+		return c, err
+	}
+
+	klog.Infof("create the clientset based on the apiserver address(%s).", o.ApiserverAddr)
+	c.Client, err = kubeutil.CreateClientSetApiserverAddr(o.ApiserverAddr)
+	return c, err
+}
diff --git a/cmd/yurt-tunnel-agent/app/start.go b/cmd/yurt-tunnel-agent/app/start.go
new file mode 100644
index 00000000..860113ff
--- /dev/null
+++ b/cmd/yurt-tunnel-agent/app/start.go
@@ -0,0 +1,106 @@
+/*
+Copyright 2020 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"fmt"
+
+	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-agent/app/config"
+	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-agent/app/options"
+	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/agent"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/pki"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/pki/certmanager"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/server/serveraddr"
+
+	"github.com/spf13/cobra"
+	"k8s.io/client-go/util/certificate"
+	"k8s.io/klog/v2"
+)
+
+// NewYurttunnelAgentCommand creates a new yurttunnel-agent command
+func NewYurttunnelAgentCommand(stopCh <-chan struct{}) *cobra.Command {
+	agentOptions := options.NewAgentOptions()
+
+	cmd := &cobra.Command{
+		Short: fmt.Sprintf("Launch %s", projectinfo.GetAgentName()),
+		RunE: func(c *cobra.Command, args []string) error {
+			if agentOptions.Version {
+				fmt.Printf("%s: %#v\n", projectinfo.GetAgentName(), projectinfo.Get())
+				return nil
+			}
+			klog.Infof("%s version: %#v\n", projectinfo.GetAgentName(), projectinfo.Get())
+
+			if err := agentOptions.Validate(); err != nil {
+				return err
+			}
+
+			cfg, err := agentOptions.Config()
+			if err != nil {
+				return err
+			}
+			if err := Run(cfg.Complete(), stopCh); err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+
+	agentOptions.AddFlags(cmd.Flags())
+	return cmd
+}
+
+// Run starts the yurttunel-agent
+func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
+	var (
+		tunnelServerAddr string
+		err              error
+		agentCertMgr     certificate.Manager
+	)
+
+	// 1. get the address of the yurttunnel-server
+	tunnelServerAddr = cfg.TunnelServerAddr
+	if tunnelServerAddr == "" {
+		if tunnelServerAddr, err = serveraddr.GetTunnelServerAddr(cfg.Client); err != nil {
+			return err
+		}
+	}
+	klog.Infof("%s address: %s", projectinfo.GetServerName(), tunnelServerAddr)
+
+	// 2. create a certificate manager
+	agentCertMgr, err =
+		certmanager.NewYurttunnelAgentCertManager(cfg.Client)
+	if err != nil {
+		return err
+	}
+	agentCertMgr.Start()
+
+	// 3. generate a TLS configuration for securing the connection to server
+	tlsCfg, err := pki.GenTLSConfigUseCertMgrAndCA(agentCertMgr,
+		tunnelServerAddr, constants.YurttunnelCAFile)
+	if err != nil {
+		return err
+	}
+
+	// 4. start the yurttunnel-agent
+	ta := agent.NewTunnelAgent(tlsCfg, tunnelServerAddr, cfg.NodeName, cfg.AgentIdentifiers)
+	ta.Run(stopCh)
+
+	<-stopCh
+	return nil
+}
diff --git a/cmd/yurt-tunnel-server/app/config/config.go b/cmd/yurt-tunnel-server/app/config/config.go
new file mode 100644
index 00000000..76793245
--- /dev/null
+++ b/cmd/yurt-tunnel-server/app/config/config.go
@@ -0,0 +1,63 @@
+/*
+Copyright 2021 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"crypto/x509"
+	"net"
+
+	"k8s.io/client-go/informers"
+	"k8s.io/client-go/kubernetes"
+)
+
+// Config is the main context object for yurttunel-server
+type Config struct {
+	EgressSelectorEnabled       bool
+	EnableIptables              bool
+	IptablesSyncPeriod          int
+	CertDNSNames                []string
+	CertIPs                     []net.IP
+	ListenAddrForAgent          string
+	ListenAddrForMaster         string
+	ListenInsecureAddrForMaster string
+	RootCert                    *x509.CertPool
+	Client                      kubernetes.Interface
+	SharedInformerFactory       informers.SharedInformerFactory
+	ServerCount                 int
+	ProxyStrategy               string
+	InterceptorServerUDSFile    string
+}
+
+type completedConfig struct {
+	*Config
+}
+
+// CompletedConfig same as Config, just to swap private object.
+type CompletedConfig struct {
+	// Embed a private pointer that cannot be instantiated outside of this package.
+	*completedConfig
+}
+
+// Complete fills in any fields not set that are required to have valid data. It's mutating the receiver.
+func (c *Config) Complete() *CompletedConfig {
+	cc := completedConfig{c}
+
+	if cc.InterceptorServerUDSFile == "" {
+		cc.InterceptorServerUDSFile = "/tmp/interceptor-proxier.sock"
+	}
+	return &CompletedConfig{&cc}
+}
diff --git a/cmd/yurt-tunnel-server/app/options/options.go b/cmd/yurt-tunnel-server/app/options/options.go
new file mode 100644
index 00000000..706312a8
--- /dev/null
+++ b/cmd/yurt-tunnel-server/app/options/options.go
@@ -0,0 +1,146 @@
+/*
+Copyright 2021 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"net"
+	"strings"
+	"time"
+
+	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app/config"
+	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
+	kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/pki"
+
+	"github.com/spf13/pflag"
+	"k8s.io/client-go/informers"
+	"k8s.io/klog/v2"
+	"sigs.k8s.io/apiserver-network-proxy/pkg/server"
+)
+
+// ServerOptions has the information that required by the yurttunel-server
+type ServerOptions struct {
+	KubeConfig             string
+	BindAddr               string
+	InsecureBindAddr       string
+	CertDNSNames           string
+	CertIPs                string
+	Version                bool
+	EnableIptables         bool
+	EgressSelectorEnabled  bool
+	IptablesSyncPeriod     int
+	TunnelAgentConnectPort string
+	SecurePort             string
+	InsecurePort           string
+	ServerCount            int
+	ProxyStrategy          string
+}
+
+// NewServerOptions creates a new ServerOptions
+func NewServerOptions() *ServerOptions {
+	o := &ServerOptions{
+		BindAddr:               "0.0.0.0",
+		InsecureBindAddr:       "127.0.0.1",
+		EnableIptables:         true,
+		IptablesSyncPeriod:     60,
+		ServerCount:            1,
+		TunnelAgentConnectPort: constants.YurttunnelServerAgentPort,
+		SecurePort:             constants.YurttunnelServerMasterPort,
+		InsecurePort:           constants.YurttunnelServerMasterInsecurePort,
+		ProxyStrategy:          string(server.ProxyStrategyDestHost),
+	}
+	return o
+}
+
+// Validate validates the YurttunnelServerOptions
+func (o *ServerOptions) Validate() error {
+	if len(o.BindAddr) == 0 {
+		return fmt.Errorf("%s's bind address can't be empty",
+			projectinfo.GetServerName())
+	}
+	return nil
+}
+
+// AddFlags returns flags for a specific yurttunnel-agent by section name
+func (o *ServerOptions) AddFlags(fs *pflag.FlagSet) {
+	fs.BoolVar(&o.Version, "version", o.Version, fmt.Sprintf("print the version information of the %s.", projectinfo.GetServerName()))
+	fs.StringVar(&o.KubeConfig, "kube-config", o.KubeConfig, "path to the kubeconfig file.")
+	fs.StringVar(&o.BindAddr, "bind-address", o.BindAddr, fmt.Sprintf("the ip address on which the %s will listen for --secure-port or --tunnel-agent-connect-port port.", projectinfo.GetServerName()))
+	fs.StringVar(&o.InsecureBindAddr, "insecure-bind-address", o.InsecureBindAddr, fmt.Sprintf("the ip address on which the %s will listen for --insecure-port port.", projectinfo.GetServerName()))
+	fs.StringVar(&o.CertDNSNames, "cert-dns-names", o.CertDNSNames, "DNS names that will be added into server's certificate. (e.g., dns1,dns2)")
+	fs.StringVar(&o.CertIPs, "cert-ips", o.CertIPs, "IPs that will be added into server's certificate. (e.g., ip1,ip2)")
+	fs.BoolVar(&o.EnableIptables, "enable-iptables", o.EnableIptables, "If allow iptable manager to set the dnat rule.")
+	fs.BoolVar(&o.EgressSelectorEnabled, "egress-selector-enable", o.EgressSelectorEnabled, "If the apiserver egress selector has been enabled.")
+	fs.IntVar(&o.IptablesSyncPeriod, "iptables-sync-period", o.IptablesSyncPeriod, "The synchronization period of the iptable manager.")
+	fs.IntVar(&o.ServerCount, "server-count", o.ServerCount, "The number of proxy server instances, should be 1 unless it is an HA server.")
+	fs.StringVar(&o.ProxyStrategy, "proxy-strategy", o.ProxyStrategy, "The strategy of proxying requests from tunnel server to agent.")
+	fs.StringVar(&o.TunnelAgentConnectPort, "tunnel-agent-connect-port", o.TunnelAgentConnectPort, "The port on which to serve tcp packets from tunnel agent")
+	fs.StringVar(&o.SecurePort, "secure-port", o.SecurePort, "The port on which to serve HTTPS requests from cloud clients like prometheus")
+	fs.StringVar(&o.InsecurePort, "insecure-port", o.InsecurePort, "The port on which to server HTTP requests from cloud clients like metrics-server")
+}
+
+func (o *ServerOptions) Config() (*config.Config, error) {
+	var err error
+	cfg := &config.Config{
+		EgressSelectorEnabled: o.EgressSelectorEnabled,
+		EnableIptables:        o.EnableIptables,
+		IptablesSyncPeriod:    o.IptablesSyncPeriod,
+		CertDNSNames:          make([]string, 0),
+		CertIPs:               make([]net.IP, 0),
+		ServerCount:           o.ServerCount,
+		ProxyStrategy:         o.ProxyStrategy,
+	}
+
+	if o.CertDNSNames != "" {
+		for _, name := range strings.Split(o.CertDNSNames, ",") {
+			cfg.CertDNSNames = append(cfg.CertDNSNames, name)
+		}
+	}
+
+	if o.CertIPs != "" {
+		for _, ipStr := range strings.Split(o.CertIPs, ",") {
+			ip := net.ParseIP(ipStr)
+			if ip != nil {
+				cfg.CertIPs = append(cfg.CertIPs, ip)
+			}
+		}
+	}
+
+	cfg.ListenAddrForAgent = net.JoinHostPort(o.BindAddr, o.TunnelAgentConnectPort)
+	cfg.ListenAddrForMaster = net.JoinHostPort(o.BindAddr, o.SecurePort)
+	cfg.ListenInsecureAddrForMaster = net.JoinHostPort(o.InsecureBindAddr, o.InsecurePort)
+
+	cfg.RootCert, err = pki.GenRootCertPool(o.KubeConfig, constants.YurttunnelCAFile)
+	if err != nil {
+		return nil, fmt.Errorf("fail to generate the rootCertPool: %s", err)
+	}
+
+	// function 'kubeutil.CreateClientSet' will try to create the clientset
+	// based on the in-cluster config if the kubeconfig is empty. As
+	// yurttunnel-server will run on the cloud, the in-cluster config should
+	// be available.
+	cfg.Client, err = kubeutil.CreateClientSet(o.KubeConfig)
+	if err != nil {
+		return nil, err
+	}
+	cfg.SharedInformerFactory = informers.NewSharedInformerFactory(cfg.Client, 10*time.Second)
+
+	klog.Infof("yurttunnel server config: %#+v", cfg)
+	return cfg, nil
+}
diff --git a/cmd/yurt-tunnel-server/app/start.go b/cmd/yurt-tunnel-server/app/start.go
new file mode 100644
index 00000000..6f54b48f
--- /dev/null
+++ b/cmd/yurt-tunnel-server/app/start.go
@@ -0,0 +1,141 @@
+/*
+Copyright 2020 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package app
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app/config"
+	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app/options"
+	"github.com/openyurtio/openyurt/pkg/projectinfo"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper/initializer"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper/wraphandler"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/iptables"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/pki"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/pki/certmanager"
+	"github.com/openyurtio/openyurt/pkg/yurttunnel/server"
+
+	"github.com/spf13/cobra"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/klog/v2"
+)
+
+// NewYurttunnelServerCommand creates a new yurttunnel-server command
+func NewYurttunnelServerCommand(stopCh <-chan struct{}) *cobra.Command {
+	serverOptions := options.NewServerOptions()
+
+	cmd := &cobra.Command{
+		Use:   "Launch " + projectinfo.GetServerName(),
+		Short: projectinfo.GetServerName() + " sends requests to " + projectinfo.GetAgentName(),
+		RunE: func(c *cobra.Command, args []string) error {
+			if serverOptions.Version {
+				fmt.Printf("%s: %#v\n", projectinfo.GetServerName(), projectinfo.Get())
+				return nil
+			}
+			klog.Infof("%s version: %#v\n", projectinfo.GetServerName(), projectinfo.Get())
+
+			if err := serverOptions.Validate(); err != nil {
+				return err
+			}
+
+			cfg, err := serverOptions.Config()
+			if err != nil {
+				return err
+			}
+			if err := Run(cfg.Complete(), stopCh); err != nil {
+				return err
+			}
+			return nil
+		},
+	}
+
+	serverOptions.AddFlags(cmd.Flags())
+
+	return cmd
+}
+
+// run starts the yurttunel-server
+func Run(cfg *config.CompletedConfig, stopCh <-chan struct{}) error {
+	// 1. start the IP table manager
+	if cfg.EnableIptables {
+		iptablesMgr := iptables.NewIptablesManager(cfg.Client,
+			cfg.SharedInformerFactory.Core().V1().Nodes(),
+			cfg.ListenAddrForMaster,
+			cfg.ListenInsecureAddrForMaster,
+			cfg.IptablesSyncPeriod)
+		if iptablesMgr == nil {
+			return fmt.Errorf("fail to create a new IptableManager")
+		}
+		go iptablesMgr.Run(stopCh)
+	}
+
+	// 2. create a certificate manager for the tunnel server and run the
+	// csr approver for both yurttunnel-server and yurttunnel-agent
+	serverCertMgr, err := certmanager.NewYurttunnelServerCertManager(cfg.Client, cfg.CertDNSNames, cfg.CertIPs, stopCh)
+	if err != nil {
+		return err
+	}
+	serverCertMgr.Start()
+	go certmanager.NewCSRApprover(cfg.Client, cfg.SharedInformerFactory.Certificates().V1beta1().CertificateSigningRequests()).
+		Run(constants.YurttunnelCSRApproverThreadiness, stopCh)
+
+	// 3. generate the TLS configuration based on the latest certificate
+	tlsCfg, err := pki.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, cfg.RootCert)
+	if err != nil {
+		return err
+	}
+
+	// 4. create handler wrappers
+	mInitializer := initializer.NewMiddlewareInitializer(cfg.SharedInformerFactory)
+	wrappers, err := wraphandler.InitHandlerWrappers(mInitializer)
+	if err != nil {
+		return err
+	}
+
+	// after all of informers are configured completed, start the shared index informer
+	cfg.SharedInformerFactory.Start(stopCh)
+
+	// 5. waiting for the certificate is generated
+	_ = wait.PollUntil(5*time.Second, func() (bool, error) {
+		// keep polling until the certificate is signed
+		if serverCertMgr.Current() != nil {
+			return true, nil
+		}
+		klog.Infof("waiting for the master to sign the %s certificate", projectinfo.GetServerName())
+		return false, nil
+	}, stopCh)
+
+	// 6. start the server
+	ts := server.NewTunnelServer(
+		cfg.EgressSelectorEnabled,
+		cfg.InterceptorServerUDSFile,
+		cfg.ListenAddrForMaster,
+		cfg.ListenInsecureAddrForMaster,
+		cfg.ListenAddrForAgent,
+		cfg.ServerCount,
+		tlsCfg,
+		wrappers,
+		cfg.ProxyStrategy)
+	if err := ts.Run(); err != nil {
+		return err
+	}
+
+	<-stopCh
+	return nil
+}
diff --git a/cmd/yurt-tunnel-server/server.go b/cmd/yurt-tunnel-server/server.go
index a4ea7373..a1cb70a4 100644
--- a/cmd/yurt-tunnel-server/server.go
+++ b/cmd/yurt-tunnel-server/server.go
@@ -19,8 +19,9 @@ package main
 import (
 	"flag"
 
+	"github.com/openyurtio/openyurt/cmd/yurt-tunnel-server/app"
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/server"
+
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/klog/v2"
 )
@@ -29,7 +30,7 @@ func main() {
 	klog.InitFlags(nil)
 	defer klog.Flush()
 
-	cmd := server.NewYurttunnelServerCommand(wait.NeverStop)
+	cmd := app.NewYurttunnelServerCommand(wait.NeverStop)
 	cmd.Flags().AddGoFlagSet(flag.CommandLine)
 	if err := cmd.Execute(); err != nil {
 		klog.Fatalf("%s failed: %s", projectinfo.GetServerName(), err)
diff --git a/cmd/yurthub/app/config/config.go b/cmd/yurthub/app/config/config.go
index b15497f1..16038084 100644
--- a/cmd/yurthub/app/config/config.go
+++ b/cmd/yurthub/app/config/config.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package config
 
 import (
diff --git a/cmd/yurthub/app/options/options.go b/cmd/yurthub/app/options/options.go
index 01d4ea39..d264b57b 100644
--- a/cmd/yurthub/app/options/options.go
+++ b/cmd/yurthub/app/options/options.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package options
 
 import (
diff --git a/cmd/yurthub/app/start.go b/cmd/yurthub/app/start.go
index 4d3f43ff..868961bb 100644
--- a/cmd/yurthub/app/start.go
+++ b/cmd/yurthub/app/start.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package app
 
 import (
diff --git a/cmd/yurthub/yurthub.go b/cmd/yurthub/yurthub.go
index 02672992..f1b7f274 100644
--- a/cmd/yurthub/yurthub.go
+++ b/cmd/yurthub/yurthub.go
@@ -1,3 +1,19 @@
+/*
+Copyright 2020 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
 package main
 
 import (
diff --git a/pkg/yurttunnel/agent/cmd.go b/pkg/yurttunnel/agent/cmd.go
deleted file mode 100644
index 8ed4c2f3..00000000
--- a/pkg/yurttunnel/agent/cmd.go
+++ /dev/null
@@ -1,210 +0,0 @@
-/*
-Copyright 2020 The OpenYurt Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package agent
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/spf13/cobra"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/client-go/util/certificate"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/apiserver-network-proxy/pkg/agent"
-
-	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
-	kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/pki"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/pki/certmanager"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/server/serveraddr"
-)
-
-const defaultKubeconfig = "/etc/kubernetes/kubelet.conf"
-
-// NewYurttunnelAgentCommand creates a new yurttunnel-agent command
-func NewYurttunnelAgentCommand(stopCh <-chan struct{}) *cobra.Command {
-	o := &YurttunnelAgentOptions{}
-
-	cmd := &cobra.Command{
-		Short: fmt.Sprintf("Launch %s", projectinfo.GetAgentName()),
-		RunE: func(c *cobra.Command, args []string) error {
-			if o.version {
-				fmt.Printf("%s: %#v\n", projectinfo.GetAgentName(), projectinfo.Get())
-				return nil
-			}
-			fmt.Printf("%s version: %#v\n", projectinfo.GetAgentName(), projectinfo.Get())
-
-			if err := o.validate(); err != nil {
-				return err
-			}
-			if err := o.complete(); err != nil {
-				return err
-			}
-			if err := o.run(stopCh); err != nil {
-				return err
-			}
-			return nil
-		},
-	}
-
-	flags := cmd.Flags()
-	flags.BoolVar(&o.version, "version", o.version,
-		"print the version information.")
-	flags.StringVar(&o.nodeName, "node-name", o.nodeName,
-		"The name of the edge node.")
-	flags.StringVar(&o.nodeIP, "node-ip", o.nodeIP,
-		"The host IP of the edge node.")
-	flags.StringVar(&o.tunnelServerAddr, "tunnelserver-addr", o.tunnelServerAddr,
-		fmt.Sprintf("The address of %s", projectinfo.GetServerName()))
-	flags.StringVar(&o.apiserverAddr, "apiserver-addr", o.tunnelServerAddr,
-		"A reachable address of the apiserver.")
-	flags.StringVar(&o.kubeConfig, "kube-config", o.kubeConfig,
-		"Path to the kubeconfig file.")
-	flags.StringVar(&o.agentIdentifiers, "agent-identifiers", o.agentIdentifiers,
-		"The identifiers of the agent, which will be used by the server when choosing agent.")
-
-	return cmd
-}
-
-// YurttunnelAgentOptions has the information that required by the
-// yurttunel-agent
-type YurttunnelAgentOptions struct {
-	nodeName         string
-	nodeIP           string
-	tunnelServerAddr string
-	apiserverAddr    string
-	kubeConfig       string
-	version          bool
-	clientset        kubernetes.Interface
-	agentIdentifiers string
-}
-
-// validate validates the YurttunnelServerOptions
-func (o *YurttunnelAgentOptions) validate() error {
-	if o.nodeName == "" {
-		o.nodeName = os.Getenv("NODE_NAME")
-		if o.nodeName == "" {
-			return errors.New("either --node-name or $NODE_NAME has to be set")
-		}
-	}
-
-	if o.nodeIP == "" {
-		o.nodeIP = os.Getenv("NODE_IP")
-		if o.nodeIP == "" {
-			return errors.New("either --node-ip or $NODE_IP has to be set")
-		}
-	}
-
-	if !agentIdentifiersAreValid(o.agentIdentifiers) {
-		return errors.New("--agent-identifiers are invalid, format should be host={node-name}")
-	}
-
-	return nil
-}
-
-// complete completes all the required options
-func (o *YurttunnelAgentOptions) complete() error {
-	var err error
-
-	if len(o.agentIdentifiers) == 0 {
-		o.agentIdentifiers = fmt.Sprintf("ipv4=%s&host=%s", o.nodeIP, o.nodeName)
-	}
-	klog.Infof("%s is set for agent identifies", o.agentIdentifiers)
-
-	if o.kubeConfig == "" && o.apiserverAddr == "" {
-		o.kubeConfig = defaultKubeconfig
-		klog.Infof("neither --kube-config nor --apiserver-addr is set, will use %s as the kubeconfig", o.kubeConfig)
-	}
-
-	if o.kubeConfig != "" {
-		klog.Infof("create the clientset based on the kubeconfig(%s).", o.kubeConfig)
-		o.clientset, err = kubeutil.CreateClientSetKubeConfig(o.kubeConfig)
-		return err
-	}
-
-	klog.Infof("create the clientset based on the apiserver address(%s).", o.apiserverAddr)
-	o.clientset, err = kubeutil.CreateClientSetApiserverAddr(o.apiserverAddr)
-	return err
-}
-
-// run starts the yurttunel-agent
-func (o *YurttunnelAgentOptions) run(stopCh <-chan struct{}) error {
-	var (
-		tunnelServerAddr string
-		err              error
-		agentCertMgr     certificate.Manager
-	)
-
-	// 1. get the address of the yurttunnel-server
-	tunnelServerAddr = o.tunnelServerAddr
-	if o.tunnelServerAddr == "" {
-		if tunnelServerAddr, err = serveraddr.GetTunnelServerAddr(o.clientset); err != nil {
-			return err
-		}
-	}
-	klog.Infof("%s address: %s", projectinfo.GetServerName(), tunnelServerAddr)
-
-	// 2. create a certificate manager
-	agentCertMgr, err =
-		certmanager.NewYurttunnelAgentCertManager(o.clientset)
-	if err != nil {
-		return err
-	}
-	agentCertMgr.Start()
-
-	// 3. generate a TLS configuration for securing the connection to server
-	tlsCfg, err := pki.GenTLSConfigUseCertMgrAndCA(agentCertMgr,
-		tunnelServerAddr, constants.YurttunnelCAFile)
-	if err != nil {
-		return err
-	}
-
-	// 4. start the yurttunnel-agent
-	ta := NewTunnelAgent(tlsCfg, tunnelServerAddr, o.nodeName, o.agentIdentifiers)
-	ta.Run(stopCh)
-
-	<-stopCh
-	return nil
-}
-
-// agentIdentifiersIsValid verify agent identifiers are valid or not.
-// and agentIdentifiers can be empty because default value will be set in complete() func.
-func agentIdentifiersAreValid(agentIdentifiers string) bool {
-	if len(agentIdentifiers) == 0 {
-		return true
-	}
-
-	entries := strings.Split(agentIdentifiers, ",")
-	for i := range entries {
-		parts := strings.Split(entries[i], "=")
-		if len(parts) != 2 {
-			return false
-		}
-
-		switch agent.IdentifierType(parts[0]) {
-		case agent.Host, agent.CIDR, agent.IPv4, agent.IPv6, agent.UID:
-			// valid agent identifier
-		default:
-			return false
-		}
-	}
-
-	return true
-}
diff --git a/pkg/yurttunnel/constants/constants.go b/pkg/yurttunnel/constants/constants.go
index 7b96c175..2aeb5993 100644
--- a/pkg/yurttunnel/constants/constants.go
+++ b/pkg/yurttunnel/constants/constants.go
@@ -17,9 +17,9 @@ limitations under the License.
 package constants
 
 const (
-	YurttunnelServerAgentPort          = 10262
-	YurttunnelServerMasterPort         = 10263
-	YurttunnelServerMasterInsecurePort = 10264
+	YurttunnelServerAgentPort          = "10262"
+	YurttunnelServerMasterPort         = "10263"
+	YurttunnelServerMasterInsecurePort = "10264"
 	YurttunnelServerServiceNs          = "kube-system"
 	YurttunnelServerServiceName        = "x-tunnel-server-svc"
 	YurttunnelServerAgentPortName      = "tcp"
diff --git a/pkg/yurttunnel/iptables/iptables.go b/pkg/yurttunnel/iptables/iptables.go
index 3b6b2476..977822e5 100644
--- a/pkg/yurttunnel/iptables/iptables.go
+++ b/pkg/yurttunnel/iptables/iptables.go
@@ -19,7 +19,6 @@ package iptables
 import (
 	"fmt"
 	"net"
-	"strconv"
 	"strings"
 	"time"
 
@@ -37,7 +36,6 @@ import (
 	utilnet "k8s.io/utils/net"
 
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
 )
 
 const (
@@ -93,6 +91,7 @@ type iptablesManager struct {
 	conntrackPath    string
 	secureDnatDest   string
 	insecureDnatDest string
+	insecurePort     string
 	lastNodesIP      []string
 	lastDnatPorts    []string
 	syncPeriod       int
@@ -103,8 +102,8 @@ type iptablesManager struct {
 // appends the rules to the iptable.
 func NewIptablesManager(client clientset.Interface,
 	nodeInformer coreinformer.NodeInformer,
-	nodeIP string,
-	insecureBindIP string,
+	listenAddr string,
+	listenInsecureAddr string,
 	syncPeriod int) IptablesManager {
 
 	protocol := iptables.ProtocolIpv4
@@ -116,18 +115,19 @@ func NewIptablesManager(client clientset.Interface,
 		syncPeriod = defaultSyncPeriod
 	}
 
-	secureDnatDest := fmt.Sprintf("%s:%d", nodeIP,
-		constants.YurttunnelServerMasterPort)
-	insecureDnatDest := fmt.Sprintf("%s:%d", insecureBindIP,
-		constants.YurttunnelServerMasterInsecurePort)
+	_, insecurePort, err := net.SplitHostPort(listenInsecureAddr)
+	if err != nil {
+		return nil
+	}
 
 	im := &iptablesManager{
 		kubeClient:       client,
 		iptables:         iptInterface,
 		execer:           execer,
 		nodeInformer:     nodeInformer,
-		secureDnatDest:   secureDnatDest,
-		insecureDnatDest: insecureDnatDest,
+		secureDnatDest:   listenAddr,
+		insecureDnatDest: listenInsecureAddr,
+		insecurePort:     insecurePort,
 		lastNodesIP:      make([]string, 0),
 		lastDnatPorts:    make([]string, 0),
 		syncPeriod:       syncPeriod,
@@ -229,7 +229,7 @@ func (im *iptablesManager) getConfiguredDnatPorts() []string {
 		// we only allowed user to add dnat rule that uses insecure port as the
 		// destination port.
 		if len(portPair) == 2 &&
-			portPair[1] == strconv.Itoa(constants.YurttunnelServerMasterInsecurePort) &&
+			portPair[1] == im.insecurePort &&
 			len(portPair[0]) != 0 {
 			if portPair[0] != "10250" && portPair[0] != "10255" {
 				ports = append(ports, portPair[0])
diff --git a/pkg/yurttunnel/pki/certmanager/certmanager.go b/pkg/yurttunnel/pki/certmanager/certmanager.go
index f760951d..f8fd4cdd 100644
--- a/pkg/yurttunnel/pki/certmanager/certmanager.go
+++ b/pkg/yurttunnel/pki/certmanager/certmanager.go
@@ -23,7 +23,6 @@ import (
 	"fmt"
 	"net"
 	"os"
-	"strings"
 	"time"
 
 	"github.com/openyurtio/openyurt/pkg/projectinfo"
@@ -42,8 +41,8 @@ import (
 // the yurttunnel-server
 func NewYurttunnelServerCertManager(
 	clientset kubernetes.Interface,
-	clCertNames,
-	clIPs string,
+	clCertNames []string,
+	clIPs []net.IP,
 	stopCh <-chan struct{}) (certificate.Manager, error) {
 	// get server DNS names and IPs
 	var (
@@ -60,14 +59,9 @@ func NewYurttunnelServerCertManager(
 		return false, nil
 	}, stopCh)
 	// add user specified DNS anems and IP addresses
-	if clCertNames != "" {
-		dnsNames = append(dnsNames, strings.Split(clCertNames, ",")...)
-	}
-	if clIPs != "" {
-		for _, ipstr := range strings.Split(clIPs, ",") {
-			ips = append(ips, net.ParseIP(ipstr))
-		}
-	}
+	dnsNames = append(dnsNames, clCertNames...)
+	ips = append(ips, clIPs...)
+
 	return newCertManager(
 		clientset,
 		projectinfo.GetServerName(),
diff --git a/pkg/yurttunnel/server/cmd.go b/pkg/yurttunnel/server/cmd.go
deleted file mode 100644
index 0b9fc22e..00000000
--- a/pkg/yurttunnel/server/cmd.go
+++ /dev/null
@@ -1,249 +0,0 @@
-/*
-Copyright 2020 The OpenYurt Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package server
-
-import (
-	"fmt"
-	"time"
-
-	"github.com/openyurtio/openyurt/pkg/projectinfo"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/constants"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper/initializer"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/handlerwrapper/wraphandler"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/iptables"
-	kubeutil "github.com/openyurtio/openyurt/pkg/yurttunnel/kubernetes"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/pki"
-	"github.com/openyurtio/openyurt/pkg/yurttunnel/pki/certmanager"
-	"github.com/spf13/cobra"
-
-	"k8s.io/apimachinery/pkg/util/wait"
-	"k8s.io/client-go/informers"
-	"k8s.io/client-go/kubernetes"
-	"k8s.io/klog/v2"
-	"sigs.k8s.io/apiserver-network-proxy/pkg/server"
-)
-
-// NewYurttunnelServerCommand creates a new yurttunnel-server command
-func NewYurttunnelServerCommand(stopCh <-chan struct{}) *cobra.Command {
-	o := NewYurttunnelServerOptions()
-
-	cmd := &cobra.Command{
-		Use:   "Launch " + projectinfo.GetServerName(),
-		Short: projectinfo.GetServerName() + " sends requests to " + projectinfo.GetAgentName(),
-		RunE: func(c *cobra.Command, args []string) error {
-			if o.version {
-				fmt.Printf("%s: %#v\n", projectinfo.GetServerName(), projectinfo.Get())
-				return nil
-			}
-			fmt.Printf("%s version: %#v\n", projectinfo.GetServerName(), projectinfo.Get())
-
-			if err := o.validate(); err != nil {
-				return err
-			}
-			if err := o.complete(); err != nil {
-				return err
-			}
-			if err := o.run(stopCh); err != nil {
-				return err
-			}
-			return nil
-		},
-	}
-
-	flags := cmd.Flags()
-	flags.BoolVar(&o.version, "version", o.version,
-		fmt.Sprintf("print the version information of the %s.",
-			projectinfo.GetServerName()))
-	flags.StringVar(&o.kubeConfig, "kube-config", o.kubeConfig,
-		"path to the kubeconfig file.")
-	flags.StringVar(&o.bindAddr, "bind-address", o.bindAddr,
-		fmt.Sprintf("the ip address on which the %s will listen.",
-			projectinfo.GetServerName()))
-	flags.StringVar(&o.insecureBindAddr, "insecure-bind-address", o.insecureBindAddr,
-		fmt.Sprintf("the ip address on which the %s will listen without tls.",
-			projectinfo.GetServerName()))
-	flags.StringVar(&o.certDNSNames, "cert-dns-names", o.certDNSNames,
-		"DNS names that will be added into server's certificate. (e.g., dns1,dns2)")
-	flags.StringVar(&o.certIPs, "cert-ips", o.certIPs,
-		"IPs that will be added into server's certificate. (e.g., ip1,ip2)")
-	flags.BoolVar(&o.enableIptables, "enable-iptables", o.enableIptables,
-		"If allow iptable manager to set the dnat rule.")
-	flags.BoolVar(&o.egressSelectorEnabled, "egress-selector-enable", o.egressSelectorEnabled,
-		"If the apiserver egress selector has been enabled.")
-	flags.IntVar(&o.iptablesSyncPeriod, "iptables-sync-period", o.iptablesSyncPeriod,
-		"The synchronization period of the iptable manager.")
-	flags.IntVar(&o.serverCount, "server-count", o.serverCount,
-		"The number of proxy server instances, should be 1 unless it is an HA server.")
-	flags.StringVar(&o.proxyStrategy, "proxy-strategy", o.proxyStrategy,
-		"The strategy of proxying requests from tunnel server to agent.")
-
-	return cmd
-}
-
-// YurttunnelServerOptions has the information that required by the
-// yurttunel-server
-type YurttunnelServerOptions struct {
-	kubeConfig               string
-	bindAddr                 string
-	insecureBindAddr         string
-	certDNSNames             string
-	certIPs                  string
-	version                  bool
-	enableIptables           bool
-	egressSelectorEnabled    bool
-	iptablesSyncPeriod       int
-	serverAgentPort          int
-	serverMasterPort         int
-	serverMasterInsecurePort int
-	serverCount              int
-	interceptorServerUDSFile string
-	serverAgentAddr          string
-	serverMasterAddr         string
-	serverMasterInsecureAddr string
-	clientset                kubernetes.Interface
-	sharedInformerFactory    informers.SharedInformerFactory
-	proxyStrategy            string
-}
-
-// NewYurttunnelServerOptions creates a new YurtNewYurttunnelServerOptions
-func NewYurttunnelServerOptions() *YurttunnelServerOptions {
-	o := &YurttunnelServerOptions{
-		bindAddr:                 "0.0.0.0",
-		insecureBindAddr:         "127.0.0.1",
-		enableIptables:           true,
-		iptablesSyncPeriod:       60,
-		serverCount:              1,
-		serverAgentPort:          constants.YurttunnelServerAgentPort,
-		serverMasterPort:         constants.YurttunnelServerMasterPort,
-		serverMasterInsecurePort: constants.YurttunnelServerMasterInsecurePort,
-		interceptorServerUDSFile: "/tmp/interceptor-proxier.sock",
-		proxyStrategy:            string(server.ProxyStrategyDestHost),
-	}
-	return o
-}
-
-// validate validates the YurttunnelServerOptions
-func (o *YurttunnelServerOptions) validate() error {
-	if len(o.bindAddr) == 0 {
-		return fmt.Errorf("%s's bind address can't be empty",
-			projectinfo.GetServerName())
-	}
-	return nil
-}
-
-// complete completes all the required options
-func (o *YurttunnelServerOptions) complete() error {
-	o.serverAgentAddr = fmt.Sprintf("%s:%d", o.bindAddr, o.serverAgentPort)
-	o.serverMasterAddr = fmt.Sprintf("%s:%d", o.bindAddr, o.serverMasterPort)
-	o.serverMasterInsecureAddr = fmt.Sprintf("%s:%d", o.insecureBindAddr, o.serverMasterInsecurePort)
-	klog.Infof("server will accept %s requests at: %s, "+
-		"server will accept master https requests at: %s"+
-		"server will accept master http request at: %s",
-		projectinfo.GetAgentName(), o.serverAgentAddr,
-		o.serverMasterAddr, o.serverMasterInsecureAddr)
-	var err error
-	// function 'kubeutil.CreateClientSet' will try to create the clientset
-	// based on the in-cluster config if the kubeconfig is empty. As
-	// yurttunnel-server will run on the cloud, the in-cluster config should
-	// be available.
-	o.clientset, err = kubeutil.CreateClientSet(o.kubeConfig)
-	if err != nil {
-		return err
-	}
-	o.sharedInformerFactory =
-		informers.NewSharedInformerFactory(o.clientset, 10*time.Second)
-	return nil
-}
-
-// run starts the yurttunel-server
-func (o *YurttunnelServerOptions) run(stopCh <-chan struct{}) error {
-	// 1. start the IP table manager
-	if o.enableIptables {
-		iptablesMgr := iptables.NewIptablesManager(o.clientset,
-			o.sharedInformerFactory.Core().V1().Nodes(),
-			o.bindAddr,
-			o.insecureBindAddr,
-			o.iptablesSyncPeriod)
-		if iptablesMgr == nil {
-			return fmt.Errorf("fail to create a new IptableManager")
-		}
-		go iptablesMgr.Run(stopCh)
-	}
-
-	// 2. create a certificate manager for the tunnel server and run the
-	// csr approver for both yurttunnel-server and yurttunnel-agent
-	serverCertMgr, err :=
-		certmanager.NewYurttunnelServerCertManager(
-			o.clientset, o.certDNSNames, o.certIPs, stopCh)
-	if err != nil {
-		return err
-	}
-	serverCertMgr.Start()
-	go certmanager.NewCSRApprover(o.clientset, o.sharedInformerFactory.Certificates().V1beta1().CertificateSigningRequests()).
-		Run(constants.YurttunnelCSRApproverThreadiness, stopCh)
-
-	// 3. generate the TLS configuration based on the latest certificate
-	rootCertPool, err := pki.GenRootCertPool(o.kubeConfig,
-		constants.YurttunnelCAFile)
-	if err != nil {
-		return fmt.Errorf("fail to generate the rootCertPool: %s", err)
-	}
-	tlsCfg, err :=
-		pki.GenTLSConfigUseCertMgrAndCertPool(serverCertMgr, rootCertPool)
-	if err != nil {
-		return err
-	}
-
-	// 4. create handler wrappers
-	mInitializer := initializer.NewMiddlewareInitializer(o.sharedInformerFactory)
-	wrappers, err := wraphandler.InitHandlerWrappers(mInitializer)
-	if err != nil {
-		return err
-	}
-
-	// after all of informers are configured completed, start the shared index informer
-	o.sharedInformerFactory.Start(stopCh)
-
-	// 5. waiting for the certificate is generated
-	_ = wait.PollUntil(5*time.Second, func() (bool, error) {
-		// keep polling until the certificate is signed
-		if serverCertMgr.Current() != nil {
-			return true, nil
-		}
-		klog.Infof("waiting for the master to sign the %s certificate",
-			projectinfo.GetServerName())
-		return false, nil
-	}, stopCh)
-
-	// 6. start the server
-	ts := NewTunnelServer(
-		o.egressSelectorEnabled,
-		o.interceptorServerUDSFile,
-		o.serverMasterAddr,
-		o.serverMasterInsecureAddr,
-		o.serverAgentAddr,
-		o.serverCount,
-		tlsCfg,
-		wrappers,
-		o.proxyStrategy)
-	if err := ts.Run(); err != nil {
-		return err
-	}
-
-	<-stopCh
-	return nil
-}


Test patch:
diff --git a/pkg/yurttunnel/agent/cmd_test.go b/cmd/yurt-tunnel-agent/app/options/options_test.go
similarity index 98%
rename from pkg/yurttunnel/agent/cmd_test.go
rename to cmd/yurt-tunnel-agent/app/options/options_test.go
index cdac5e3f..50af4b07 100644
--- a/pkg/yurttunnel/agent/cmd_test.go
+++ b/cmd/yurt-tunnel-agent/app/options/options_test.go
@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package agent
+package options
 
 import "testing"