Repository: matrixorigin/matrixone
Pull request #4289 (2022-07-30 14:51:29)

logservice: updated logservice configurations
use []string in toml files, no more semicoln separated strings

remove the HAKeeperReplicaID field in bootstrap config

**What type of PR is this?**

- [ ] API-change
- [ ] BUG
- [x] Improvement
- [ ] Documentation
- [ ] Feature
- [ ] Test and CI
- [ ] Code Refactoring

**Which issue(s) this PR fixes:**

issue #

**What this PR does / why we need it:**

Not Available

**Special notes for your reviewer:**

Not Available

**Additional documentation (e.g. design docs, usage docs, etc.):**

Not Available



Patch:
diff --git a/.gitignore b/.gitignore
index 3eaa74340..1552b4c67 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,8 @@
 .idea/
 *.o
 *.a
+*.exe
+*.lib
 *.swp
 bin/
 y.go
@@ -12,6 +14,7 @@ y.go
 tags
 system_vars_config.toml
 coverage.txt
+out.txt
 vendor/
 gen_config
 main
@@ -20,6 +23,7 @@ mo-service
 cube*/
 store/
 mo-data/
+node-*-data/
 pkg/catalog/test/
 pkg/config/system_vars.go
 pkg/config/system_vars_test.go
diff --git a/cmd/mo-service/cfg.go b/cmd/mo-service/config.go
similarity index 88%
rename from cmd/mo-service/cfg.go
rename to cmd/mo-service/config.go
index 717a49de3..2434467fd 100644
--- a/cmd/mo-service/cfg.go
+++ b/cmd/mo-service/config.go
@@ -51,6 +51,8 @@ type Config struct {
 	ServiceType string `toml:"service-type"`
 	// FileServices the config for file services
 	FileServices []fileservice.Config `toml:"fileservice"`
+
+	HAKeeperClient logservice.HAKeeperClientConfig `toml:"hakeeper-client"`
 	// DN dn service config
 	DN dnservice.Config `toml:"dn"`
 	// LogService is the config for log service
@@ -87,6 +89,7 @@ func (c *Config) validate() error {
 	return nil
 }
 
+// FIXME: fileservice created by a config instance is very strange at best
 func (c *Config) createFileService(name string) (fileservice.FileService, error) {
 	for _, cfg := range c.FileServices {
 		if strings.EqualFold(cfg.Name, name) {
@@ -95,3 +98,10 @@ func (c *Config) createFileService(name string) (fileservice.FileService, error)
 	}
 	return nil, fmt.Errorf("file service named %s not set", name)
 }
+
+func (c *Config) getLogServiceConfig() logservice.Config {
+	cfg := c.LogService
+	fmt.Printf("hakeeper client cfg: %v", c.HAKeeperClient)
+	cfg.HAKeeperClientConfig = c.HAKeeperClient
+	return cfg
+}
diff --git a/cmd/mo-service/cfg_test.go b/cmd/mo-service/config_test.go
similarity index 100%
rename from cmd/mo-service/cfg_test.go
rename to cmd/mo-service/config_test.go
diff --git a/cmd/mo-service/main.go b/cmd/mo-service/main.go
index fee22b6d2..600044053 100644
--- a/cmd/mo-service/main.go
+++ b/cmd/mo-service/main.go
@@ -99,7 +99,8 @@ func startDNService(cfg *Config, stopper *stopper.Stopper) error {
 }
 
 func startLogService(cfg *Config, stopper *stopper.Stopper) error {
-	s, err := logservice.NewService(cfg.LogService)
+	lscfg := cfg.getLogServiceConfig()
+	s, err := logservice.NewService(lscfg)
 	if err != nil {
 		panic(err)
 	}
diff --git a/etc/bootstrap-example/log-node-1.toml b/etc/bootstrap-example/log-node-1.toml
index 72e7b5cb2..97a89fb6f 100644
--- a/etc/bootstrap-example/log-node-1.toml
+++ b/etc/bootstrap-example/log-node-1.toml
@@ -13,15 +13,24 @@ data-dir = "node-1-data"
 raft-address = "127.0.0.1:32000"
 logservice-address = "127.0.0.1:32001"
 gossip-address = "127.0.0.1:32002"
-gossip-seed-addresses = "127.0.0.1:32002;127.0.0.1:32012;127.0.0.1:32022"
+gossip-seed-addresses = [
+  "127.0.0.1:32002",
+  "127.0.0.1:32012", 
+  "127.0.0.1:32022",
+]
 
 [logservice.BootstrapConfig]
 bootstrap-cluster = true
 num-of-log-shards = 1
 num-of-dn-shards = 1
 num-of-log-shard-replicas = 3
-init-hakeeper-members = "131072:7c4dccb4-4d3c-41f8-b482-5251dc7a41bf;131073:8c4dccb4-4d3c-41f8-b482-5251dc7a41bf;131074:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"
-hakeeper-replica-id = 131072
+init-hakeeper-members = [
+  "131072:7c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+  "131073:8c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+  "131074:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+]
 
-[logservice.HAKeeperClientConfig]
-hakeeper-service-addresses = "127.0.0.1:32001"
+[hakeeper-client]
+service-addresses = [
+  "127.0.0.1:32001",
+]
diff --git a/etc/bootstrap-example/log-node-2.toml b/etc/bootstrap-example/log-node-2.toml
index 507e4813a..feecc3734 100644
--- a/etc/bootstrap-example/log-node-2.toml
+++ b/etc/bootstrap-example/log-node-2.toml
@@ -13,15 +13,24 @@ data-dir = "node-2-data"
 raft-address = "127.0.0.1:32010"
 logservice-address = "127.0.0.1:32011"
 gossip-address = "127.0.0.1:32012"
-gossip-seed-addresses = "127.0.0.1:32002;127.0.0.1:32012;127.0.0.1:32022"
+gossip-seed-addresses = [
+  "127.0.0.1:32002",
+  "127.0.0.1:32012",
+  "127.0.0.1:32022",
+]
 
 [logservice.BootstrapConfig]
 bootstrap-cluster = true
 num-of-log-shards = 1
 num-of-dn-shards = 1
 num-of-log-shard-replicas = 3
-init-hakeeper-members = "131072:7c4dccb4-4d3c-41f8-b482-5251dc7a41bf;131073:8c4dccb4-4d3c-41f8-b482-5251dc7a41bf;131074:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"
-hakeeper-replica-id = 131073
+init-hakeeper-members = [
+  "131072:7c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+  "131073:8c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+  "131074:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+]
 
-[logservice.HAKeeperClientConfig]
-hakeeper-service-addresses = "127.0.0.1:32001"
+[hakeeper-client]
+service-addresses = [
+  "127.0.0.1:32001",
+]
diff --git a/etc/bootstrap-example/log-node-3.toml b/etc/bootstrap-example/log-node-3.toml
index 39ac65d52..79570f255 100644
--- a/etc/bootstrap-example/log-node-3.toml
+++ b/etc/bootstrap-example/log-node-3.toml
@@ -13,15 +13,24 @@ uuid = "9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"
 raft-address = "127.0.0.1:32020"
 logservice-address = "127.0.0.1:32021"
 gossip-address = "127.0.0.1:32022"
-gossip-seed-addresses = "127.0.0.1:32002;127.0.0.1:32012;127.0.0.1:32022"
+gossip-seed-addresses = [
+  "127.0.0.1:32002",
+  "127.0.0.1:32012",
+  "127.0.0.1:32022",
+]
 
 [logservice.BootstrapConfig]
 bootstrap-cluster = true
 num-of-log-shards = 1
 num-of-dn-shards = 1
 num-of-log-shard-replicas = 3
-init-hakeeper-members = "131072:7c4dccb4-4d3c-41f8-b482-5251dc7a41bf;131073:8c4dccb4-4d3c-41f8-b482-5251dc7a41bf;131074:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"
-hakeeper-replica-id = 131074
+init-hakeeper-members = [
+  "131072:7c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+  "131073:8c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+  "131074:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+]
 
-[logservice.HAKeeperClientConfig]
-hakeeper-service-addresses = "127.0.0.1:32001"
+[hakeeper-client]
+service-addresses = [
+  "127.0.0.1:32001",
+]
diff --git a/pkg/logservice/config.go b/pkg/logservice/config.go
index c40e1dfaa..8d223a686 100644
--- a/pkg/logservice/config.go
+++ b/pkg/logservice/config.go
@@ -83,9 +83,9 @@ type Config struct {
 	GossipAddress string `toml:"gossip-address"`
 	// GossipListenAddress is the local listen address of the GossipAddress
 	GossipListenAddress string `toml:"gossip-listen-address"`
-	// GossipSeedAddresses is list of semicolon separated seed addresses that
-	// are used for introducing the local node into the gossip network.
-	GossipSeedAddresses string `toml:"gossip-seed-addresses"`
+	// GossipSeedAddresses is list of seed addresses that are used for
+	// introducing the local node into the gossip network.
+	GossipSeedAddresses []string `toml:"gossip-seed-addresses"`
 	// HeartbeatInterval is the interval of how often log service node should be
 	// sending heartbeat message to the HAKeeper.
 	HeartbeatInterval toml.Duration `toml:"logservice-heartbeat-interval"`
@@ -101,6 +101,8 @@ type Config struct {
 	// initial HAKeeper replicas during bootstrapping.
 	BootstrapConfig struct {
 		// BootstrapCluster indicates whether the cluster should be bootstrapped.
+		// Note the bootstrapping procedure will only be executed if BootstrapCluster
+		// is true and Config.UUID is found in Config.BootstrapConfig.InitHAKeeperMembers.
 		BootstrapCluster bool `toml:"bootstrap-cluster"`
 		// NumOfLogShards defines the number of Log shards in the initial deployment.
 		NumOfLogShards uint64 `toml:"num-of-log-shards"`
@@ -111,23 +113,23 @@ type Config struct {
 		// Log Stores, including Log Service shards and the HAKeeper.
 		NumOfLogShardReplicas uint64 `toml:"num-of-log-shard-replicas"`
 		// InitHAKeeperMembers defines the initial members of the HAKeeper as a list
-		// of semicolon separated HAKeeper replicaID and UUID pairs. For example,
+		// of HAKeeper replicaID and UUID pairs. For example,
 		// when the initial HAKeeper members are
 		// replica with replica ID 101 running on Log Store uuid1
 		// replica with replica ID 102 running on Log Store uuid2
 		// replica with replica ID 103 running on Log Store uuid3
 		// the InitHAKeeperMembers string value should be
-		// "101:uuid1;102:uuid2;103:uuid3"
+		// []string{"101:uuid1", "102:uuid2", "103:uuid3"}
 		// Note that these initial HAKeeper replica IDs must be assigned by k8s
 		// from the range [K8SIDRangeStart, K8SIDRangeEnd) as defined in pkg/hakeeper.
 		// All uuid values are assigned by k8s, they are used to uniquely identify
 		// CN/DN/Log stores.
-		InitHAKeeperMembers string `toml:"init-hakeeper-members"`
-		// HAKeeperReplicaID is the replica ID of the HAKeeper replica to be started
-		// on the local Log Store during bootstrapping. Note that this replica is an
-		// initial member of the HAKeeper so the replicaID should exist in
-		// InitHAKeeperMembers.
-		HAKeeperReplicaID uint64 `toml:"hakeeper-replica-id"`
+		// Config.UUID and Config.BootstrapConfig values are considered together to
+		// figure out what is the replica ID of the initial HAKeeper replica. That
+		// is when Config.UUID is found in InitHAKeeperMembers, then the corresponding
+		// replica ID value will be used to launch a HAKeeper replica on the Log
+		// Service instance.
+		InitHAKeeperMembers []string `toml:"init-hakeeper-members"`
 	}
 
 	HAKeeperConfig struct {
@@ -145,29 +147,13 @@ type Config struct {
 		DnStoreTimeout toml.Duration `toml:"dn-store-timeout"`
 	}
 
-	HAKeeperClientConfig struct {
-		// DiscoveryAddress is the Log Service discovery address provided by k8s.
-		DiscoveryAddress string `toml:"hakeeper-discovery-address"`
-		// ServiceAddresses is a list of semicolon separated well known Log Services'
-		// service addresses.
-		ServiceAddresses string `toml:"hakeeper-service-addresses"`
-	}
-
+	// HAKeeperClientConfig is the config for HAKeeperClient
+	HAKeeperClientConfig HAKeeperClientConfig
 	// DisableWorkers disables the HAKeeper ticker and HAKeeper client in tests.
 	// Never set this field to true in production
 	DisableWorkers bool
 }
 
-// GetGossipSeedAddresses returns a slice of gossip seed addresses split from
-// the GossipSeedAddresses field.
-func (c *Config) GetGossipSeedAddresses() []string {
-	return splitAddresses(c.GossipSeedAddresses)
-}
-
-func (c *Config) GetHAKeeperServiceAddresses() []string {
-	return splitAddresses(c.HAKeeperClientConfig.ServiceAddresses)
-}
-
 func (c *Config) GetHAKeeperConfig() hakeeper.Config {
 	return hakeeper.Config{
 		TickPerSecond:   c.HAKeeperConfig.TickPerSecond,
@@ -177,17 +163,35 @@ func (c *Config) GetHAKeeperConfig() hakeeper.Config {
 }
 
 func (c *Config) GetHAKeeperClientConfig() HAKeeperClientConfig {
+	saddr := make([]string, 0)
+	saddr = append(saddr, c.HAKeeperClientConfig.ServiceAddresses...)
 	return HAKeeperClientConfig{
 		DiscoveryAddress: c.HAKeeperClientConfig.DiscoveryAddress,
-		ServiceAddresses: c.GetHAKeeperServiceAddresses(),
+		ServiceAddresses: saddr,
 	}
 }
 
+// returns replica ID of the HAKeeper replica and a boolean indicating whether
+// we should run the bootstrap procedure.
+func (c *Config) Bootstrapping() (uint64, bool) {
+	if !c.BootstrapConfig.BootstrapCluster {
+		return 0, false
+	}
+	members, err := c.GetInitHAKeeperMembers()
+	if err != nil {
+		return 0, false
+	}
+	for replicaID, uuid := range members {
+		if uuid == c.UUID {
+			return replicaID, true
+		}
+	}
+	return 0, false
+}
+
 func (c *Config) GetInitHAKeeperMembers() (map[uint64]dragonboat.Target, error) {
-	v := strings.TrimSpace(c.BootstrapConfig.InitHAKeeperMembers)
-	replicas := strings.Split(v, ";")
 	result := make(map[uint64]dragonboat.Target)
-	for _, pair := range replicas {
+	for _, pair := range c.BootstrapConfig.InitHAKeeperMembers {
 		pair = strings.TrimSpace(pair)
 		parts := strings.Split(pair, ":")
 		if len(parts) == 2 {
@@ -270,9 +274,6 @@ func (c *Config) Validate() error {
 		if uint64(len(members)) != c.BootstrapConfig.NumOfLogShardReplicas {
 			return errors.Wrapf(ErrInvalidConfig, "InitHAKeeperMembers and NumOfLogShardReplicas not match")
 		}
-		if _, ok := members[c.BootstrapConfig.HAKeeperReplicaID]; !ok {
-			return errors.Wrapf(ErrInvalidConfig, "HAKeeperReplicaID not a part of InitHAKeeperMembers")
-		}
 	}
 
 	return nil
diff --git a/pkg/logservice/service_bootstrap.go b/pkg/logservice/service_bootstrap.go
index cbd43893f..274b56486 100644
--- a/pkg/logservice/service_bootstrap.go
+++ b/pkg/logservice/service_bootstrap.go
@@ -22,11 +22,14 @@ import (
 )
 
 func (s *Service) BootstrapHAKeeper(ctx context.Context, cfg Config) error {
+	replicaID, bootstrapping := cfg.Bootstrapping()
+	if !bootstrapping {
+		return nil
+	}
 	members, err := cfg.GetInitHAKeeperMembers()
 	if err != nil {
 		return err
 	}
-	replicaID := cfg.BootstrapConfig.HAKeeperReplicaID
 	if err := s.store.startHAKeeperReplica(replicaID, members, false); err != nil {
 		// let's be a little bit less strict, when HAKeeper replica is already
 		// running as a result of store.startReplicas(), we just ignore the
diff --git a/pkg/logservice/store.go b/pkg/logservice/store.go
index 8686e1ac0..6846c8ef2 100644
--- a/pkg/logservice/store.go
+++ b/pkg/logservice/store.go
@@ -85,7 +85,7 @@ func getNodeHostConfig(cfg Config) config.NodeHostConfig {
 		Gossip: config.GossipConfig{
 			BindAddress:      cfg.GossipListenAddress,
 			AdvertiseAddress: cfg.GossipAddress,
-			Seed:             cfg.GetGossipSeedAddresses(),
+			Seed:             cfg.GossipSeedAddresses,
 			Meta:             meta.marshal(),
 		},
 	}


Test patch:
diff --git a/etc/log-minimal-test.toml b/etc/log-minimal-test.toml
index 8a3cb3396..ccac5b02e 100644
--- a/etc/log-minimal-test.toml
+++ b/etc/log-minimal-test.toml
@@ -9,15 +9,20 @@ max-size = 512
 [logservice]
 deployment-id = 1
 uuid = "9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"
-gossip-seed-addresses = "127.0.0.1:32002"
+gossip-seed-addresses = [
+  "127.0.0.1:32002",
+]
 
 [logservice.BootstrapConfig]
 bootstrap-cluster = true
 num-of-log-shards = 1
 num-of-dn-shards = 1
 num-of-log-shard-replicas = 1
-init-hakeeper-members = "131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"
-hakeeper-replica-id = 131072
+init-hakeeper-members = [
+  "131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
+]
 
-[logservice.HAKeeperClientConfig]
-hakeeper-service-addresses = "127.0.0.1:32000"
+[hakeeper-client]
+service-addresses = [
+  "127.0.0.1:32000",
+]
diff --git a/pkg/logservice/config_test.go b/pkg/logservice/config_test.go
index ca7877cd0..03009e52a 100644
--- a/pkg/logservice/config_test.go
+++ b/pkg/logservice/config_test.go
@@ -35,7 +35,7 @@ func getTestConfig() Config {
 		RaftListenAddress:    "localhost:9001",
 		GossipAddress:        "localhost:9002",
 		GossipListenAddress:  "localhost:9002",
-		GossipSeedAddresses:  "localhost:9002",
+		GossipSeedAddresses:  []string{"localhost:9002"},
 	}
 	c.Fill()
 	return c
@@ -65,7 +65,7 @@ func TestSplitAddress(t *testing.T) {
 
 func TestGetInitHAKeeperMembers(t *testing.T) {
 	cfg1 := Config{}
-	cfg1.BootstrapConfig.InitHAKeeperMembers = " 131072 :9c4dccb4-4d3c-41f8-b482-5251dc7a41bf "
+	cfg1.BootstrapConfig.InitHAKeeperMembers = []string{" 131072 :9c4dccb4-4d3c-41f8-b482-5251dc7a41bf "}
 	result, err := cfg1.GetInitHAKeeperMembers()
 	require.NoError(t, err)
 	assert.Equal(t, 1, len(result))
@@ -74,7 +74,7 @@ func TestGetInitHAKeeperMembers(t *testing.T) {
 	assert.Equal(t, "9c4dccb4-4d3c-41f8-b482-5251dc7a41bf", v)
 
 	cfg2 := Config{}
-	cfg2.BootstrapConfig.InitHAKeeperMembers = "131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf;131073:9c4dccb4-4d3c-41f8-b482-5251dc7a41be"
+	cfg2.BootstrapConfig.InitHAKeeperMembers = []string{"131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf", "131073:9c4dccb4-4d3c-41f8-b482-5251dc7a41be"}
 	result, err = cfg2.GetInitHAKeeperMembers()
 	require.NoError(t, err)
 	assert.Equal(t, 2, len(result))
@@ -85,13 +85,13 @@ func TestGetInitHAKeeperMembers(t *testing.T) {
 	assert.Equal(t, "9c4dccb4-4d3c-41f8-b482-5251dc7a41bf", v1)
 	assert.Equal(t, "9c4dccb4-4d3c-41f8-b482-5251dc7a41be", v2)
 
-	tests := []string{
-		"131071:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
-		"262144:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
-		"262145:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf",
-		"131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41b",
-		"131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf;",
-		"131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf;1:1",
+	tests := [][]string{
+		{"131071:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"},
+		{"262144:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"},
+		{"262145:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"},
+		{"131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41b"},
+		{"131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf", ""},
+		{"131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf", "1:1"},
 	}
 
 	for _, v := range tests {
@@ -102,14 +102,6 @@ func TestGetInitHAKeeperMembers(t *testing.T) {
 	}
 }
 
-func TestGetGossipSeedAddresses(t *testing.T) {
-	cfg := Config{
-		GossipSeedAddresses: "localhost:9000;localhost:9001 ; localhost:9002 ",
-	}
-	values := cfg.GetGossipSeedAddresses()
-	assert.Equal(t, []string{"localhost:9000", "localhost:9001", "localhost:9002"}, values)
-}
-
 func TestConfigCanBeValidated(t *testing.T) {
 	c := getTestConfig()
 	assert.NoError(t, c.Validate())
@@ -131,7 +123,7 @@ func TestConfigCanBeValidated(t *testing.T) {
 	assert.True(t, errors.Is(c4.Validate(), ErrInvalidConfig))
 
 	c5 := c
-	c5.GossipSeedAddresses = ""
+	c5.GossipSeedAddresses = []string{}
 	assert.True(t, errors.Is(c5.Validate(), ErrInvalidConfig))
 }
 
@@ -143,8 +135,7 @@ func TestBootstrapConfigCanBeValidated(t *testing.T) {
 	c.BootstrapConfig.NumOfLogShards = 3
 	c.BootstrapConfig.NumOfDNShards = 3
 	c.BootstrapConfig.NumOfLogShardReplicas = 1
-	c.BootstrapConfig.InitHAKeeperMembers = "131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"
-	c.BootstrapConfig.HAKeeperReplicaID = 131072
+	c.BootstrapConfig.InitHAKeeperMembers = []string{"131072:9c4dccb4-4d3c-41f8-b482-5251dc7a41bf"}
 	assert.NoError(t, c.Validate())
 
 	c1 := c
@@ -162,10 +153,6 @@ func TestBootstrapConfigCanBeValidated(t *testing.T) {
 	c4 := c
 	c4.BootstrapConfig.NumOfLogShardReplicas = 2
 	assert.True(t, errors.Is(c4.Validate(), ErrInvalidConfig))
-
-	c5 := c
-	c5.BootstrapConfig.HAKeeperReplicaID = 2
-	assert.True(t, errors.Is(c5.Validate(), ErrInvalidConfig))
 }
 
 func TestFillConfig(t *testing.T) {
@@ -194,7 +181,7 @@ func TestListenAddressCanBeFilled(t *testing.T) {
 		ServiceAddress:      "127.0.0.1:9002",
 		RaftAddress:         "127.0.0.1:9000",
 		GossipAddress:       "127.0.0.1:9001",
-		GossipSeedAddresses: "127.0.0.1:9011",
+		GossipSeedAddresses: []string{"127.0.0.1:9011"},
 	}
 	cfg.Fill()
 	assert.Equal(t, cfg.ServiceAddress, cfg.ServiceListenAddress)
diff --git a/pkg/logservice/hakeeper_client_test.go b/pkg/logservice/hakeeper_client_test.go
index 7ae22cdfb..9c253f36e 100644
--- a/pkg/logservice/hakeeper_client_test.go
+++ b/pkg/logservice/hakeeper_client_test.go
@@ -216,7 +216,7 @@ func testNotHAKeeperErrorIsHandled(t *testing.T, fn func(*testing.T, *managedHAK
 		ServiceAddress:      "127.0.0.1:9002",
 		RaftAddress:         "127.0.0.1:9000",
 		GossipAddress:       "127.0.0.1:9001",
-		GossipSeedAddresses: "127.0.0.1:9011",
+		GossipSeedAddresses: []string{"127.0.0.1:9011"},
 		DisableWorkers:      true,
 	}
 	cfg2 := Config{
@@ -228,7 +228,7 @@ func testNotHAKeeperErrorIsHandled(t *testing.T, fn func(*testing.T, *managedHAK
 		ServiceAddress:      "127.0.0.1:9012",
 		RaftAddress:         "127.0.0.1:9010",
 		GossipAddress:       "127.0.0.1:9011",
-		GossipSeedAddresses: "127.0.0.1:9001",
+		GossipSeedAddresses: []string{"127.0.0.1:9001"},
 		DisableWorkers:      true,
 	}
 	cfg1.Fill()
diff --git a/pkg/logservice/commands.go b/pkg/logservice/service_commands.go
similarity index 100%
rename from pkg/logservice/commands.go
rename to pkg/logservice/service_commands.go
diff --git a/pkg/logservice/commands_test.go b/pkg/logservice/service_commands_test.go
similarity index 97%
rename from pkg/logservice/commands_test.go
rename to pkg/logservice/service_commands_test.go
index a5cfd6ff4..ed573f7b0 100644
--- a/pkg/logservice/commands_test.go
+++ b/pkg/logservice/service_commands_test.go
@@ -39,11 +39,11 @@ func TestBackgroundTickAndHeartbeat(t *testing.T) {
 		ServiceAddress:      "127.0.0.1:9002",
 		RaftAddress:         "127.0.0.1:9000",
 		GossipAddress:       "127.0.0.1:9001",
-		GossipSeedAddresses: "127.0.0.1:9010",
+		GossipSeedAddresses: []string{"127.0.0.1:9010"},
 	}
 	cfg.HeartbeatInterval.Duration = 5 * time.Millisecond
 	cfg.HAKeeperTickInterval.Duration = 5 * time.Millisecond
-	cfg.HAKeeperClientConfig.ServiceAddresses = "127.0.0.1:9002"
+	cfg.HAKeeperClientConfig.ServiceAddresses = []string{"127.0.0.1:9002"}
 	cfg.Fill()
 	service, err := NewService(cfg)
 	require.NoError(t, err)
diff --git a/pkg/logservice/service_test.go b/pkg/logservice/service_test.go
index 3b5b5cce7..592e06b1c 100644
--- a/pkg/logservice/service_test.go
+++ b/pkg/logservice/service_test.go
@@ -40,7 +40,7 @@ func getServiceTestConfig() Config {
 	c := Config{
 		UUID:                 uuid.New().String(),
 		RTTMillisecond:       10,
-		GossipSeedAddresses:  "127.0.0.1:9000",
+		GossipSeedAddresses:  []string{"127.0.0.1:9000"},
 		DeploymentID:         1,
 		FS:                   vfs.NewStrictMem(),
 		ServiceListenAddress: testServiceAddress,
@@ -492,7 +492,7 @@ func TestShardInfoCanBeQueried(t *testing.T) {
 		ServiceAddress:      "127.0.0.1:9002",
 		RaftAddress:         "127.0.0.1:9000",
 		GossipAddress:       "127.0.0.1:9001",
-		GossipSeedAddresses: "127.0.0.1:9011",
+		GossipSeedAddresses: []string{"127.0.0.1:9011"},
 		DisableWorkers:      true,
 	}
 	cfg2 := Config{
@@ -504,7 +504,7 @@ func TestShardInfoCanBeQueried(t *testing.T) {
 		ServiceAddress:      "127.0.0.1:9012",
 		RaftAddress:         "127.0.0.1:9010",
 		GossipAddress:       "127.0.0.1:9011",
-		GossipSeedAddresses: "127.0.0.1:9001",
+		GossipSeedAddresses: []string{"127.0.0.1:9001"},
 		DisableWorkers:      true,
 	}
 	cfg1.Fill()
@@ -613,7 +613,7 @@ func TestGossipConvergeDelay(t *testing.T) {
 			ServiceAddress:      fmt.Sprintf("127.0.0.1:%d", 6000+10*i),
 			RaftAddress:         fmt.Sprintf("127.0.0.1:%d", 6000+10*i+1),
 			GossipAddress:       fmt.Sprintf("127.0.0.1:%d", 6000+10*i+2),
-			GossipSeedAddresses: "127.0.0.1:6002;127.0.0.1:6012",
+			GossipSeedAddresses: []string{"127.0.0.1:6002", "127.0.0.1:6012"},
 			DisableWorkers:      true,
 		}
 		configs = append(configs, cfg)
diff --git a/pkg/logservice/hakeeper_checker.go b/pkg/logservice/store_hakeeper_check.go
similarity index 100%
rename from pkg/logservice/hakeeper_checker.go
rename to pkg/logservice/store_hakeeper_check.go
diff --git a/pkg/logservice/hakeeper_checker_test.go b/pkg/logservice/store_hakeeper_check_test.go
similarity index 97%
rename from pkg/logservice/hakeeper_checker_test.go
rename to pkg/logservice/store_hakeeper_check_test.go
index bde5ef093..fd7b667cb 100644
--- a/pkg/logservice/hakeeper_checker_test.go
+++ b/pkg/logservice/store_hakeeper_check_test.go
@@ -142,7 +142,7 @@ func runHAKeeperClusterTest(t *testing.T, fn func(*testing.T, []*Service)) {
 		ServiceAddress:      "127.0.0.1:9002",
 		RaftAddress:         "127.0.0.1:9000",
 		GossipAddress:       "127.0.0.1:9001",
-		GossipSeedAddresses: "127.0.0.1:9011;127.0.0.1:9021;127.0.0.1:9031",
+		GossipSeedAddresses: []string{"127.0.0.1:9011", "127.0.0.1:9021", "127.0.0.1:9031"},
 		DisableWorkers:      true,
 	}
 	cfg1.HAKeeperConfig.TickPerSecond = 10
@@ -157,7 +157,7 @@ func runHAKeeperClusterTest(t *testing.T, fn func(*testing.T, []*Service)) {
 		ServiceAddress:      "127.0.0.1:9012",
 		RaftAddress:         "127.0.0.1:9010",
 		GossipAddress:       "127.0.0.1:9011",
-		GossipSeedAddresses: "127.0.0.1:9001;127.0.0.1:9021;127.0.0.1:9031",
+		GossipSeedAddresses: []string{"127.0.0.1:9001", "127.0.0.1:9021", "127.0.0.1:9031"},
 		DisableWorkers:      true,
 	}
 	cfg2.HAKeeperConfig.TickPerSecond = 10
@@ -172,7 +172,7 @@ func runHAKeeperClusterTest(t *testing.T, fn func(*testing.T, []*Service)) {
 		ServiceAddress:      "127.0.0.1:9022",
 		RaftAddress:         "127.0.0.1:9020",
 		GossipAddress:       "127.0.0.1:9021",
-		GossipSeedAddresses: "127.0.0.1:9001;127.0.0.1:9011;127.0.0.1:9031",
+		GossipSeedAddresses: []string{"127.0.0.1:9001", "127.0.0.1:9011", "127.0.0.1:9031"},
 		DisableWorkers:      true,
 	}
 	cfg3.HAKeeperConfig.TickPerSecond = 10
@@ -187,7 +187,7 @@ func runHAKeeperClusterTest(t *testing.T, fn func(*testing.T, []*Service)) {
 		ServiceAddress:      "127.0.0.1:9032",
 		RaftAddress:         "127.0.0.1:9030",
 		GossipAddress:       "127.0.0.1:9031",
-		GossipSeedAddresses: "127.0.0.1:9001;127.0.0.1:9011;127.0.0.1:9021",
+		GossipSeedAddresses: []string{"127.0.0.1:9001", "127.0.0.1:9011", "127.0.0.1:9021"},
 		DisableWorkers:      true,
 	}
 	cfg4.HAKeeperConfig.TickPerSecond = 10
diff --git a/pkg/logservice/store_test.go b/pkg/logservice/store_test.go
index f8a7aefad..4e5e28761 100644
--- a/pkg/logservice/store_test.go
+++ b/pkg/logservice/store_test.go
@@ -57,7 +57,7 @@ func getStoreTestConfig() Config {
 	cfg := Config{
 		UUID:                uuid.New().String(),
 		RTTMillisecond:      10,
-		GossipSeedAddresses: "127.0.0.1:9000",
+		GossipSeedAddresses: []string{"127.0.0.1:9000"},
 		DeploymentID:        1,
 		FS:                  vfs.NewStrictMem(),
 	}
@@ -420,7 +420,7 @@ func getTestStores() (*store, *store, error) {
 		ServiceAddress:      "127.0.0.1:9001",
 		RaftAddress:         "127.0.0.1:9002",
 		GossipAddress:       "127.0.0.1:9011",
-		GossipSeedAddresses: "127.0.0.1:9011;127.0.0.1:9012",
+		GossipSeedAddresses: []string{"127.0.0.1:9011", "127.0.0.1:9012"},
 	}
 	cfg1.Fill()
 	store1, err := newLogStore(cfg1)
@@ -435,7 +435,7 @@ func getTestStores() (*store, *store, error) {
 		ServiceAddress:      "127.0.0.1:9006",
 		RaftAddress:         "127.0.0.1:9007",
 		GossipAddress:       "127.0.0.1:9012",
-		GossipSeedAddresses: "127.0.0.1:9011;127.0.0.1:9012",
+		GossipSeedAddresses: []string{"127.0.0.1:9011", "127.0.0.1:9012"},
 	}
 	cfg2.Fill()
 	store2, err := newLogStore(cfg2)
diff --git a/pkg/logservice/testclient.go b/pkg/logservice/testclient.go
index 472cb6c78..47ac34a27 100644
--- a/pkg/logservice/testclient.go
+++ b/pkg/logservice/testclient.go
@@ -23,15 +23,15 @@ import (
 )
 
 func NewTestService(fs vfs.FS) (*Service, ClientConfig, error) {
-	addr := "localhost:9000"
+	addr := []string{"localhost:9000"}
 	cfg := Config{
 		UUID:                 uuid.New().String(),
 		RTTMillisecond:       10,
 		GossipSeedAddresses:  addr,
 		DeploymentID:         1,
 		FS:                   fs,
-		ServiceListenAddress: addr,
-		ServiceAddress:       addr,
+		ServiceListenAddress: addr[0],
+		ServiceAddress:       addr[0],
 		DisableWorkers:       true,
 	}
 	cfg.Fill()
@@ -55,7 +55,7 @@ func NewTestService(fs vfs.FS) (*Service, ClientConfig, error) {
 	ccfg := ClientConfig{
 		LogShardID:       1,
 		DNReplicaID:      10,
-		ServiceAddresses: []string{addr},
+		ServiceAddresses: addr,
 	}
 	return service, ccfg, nil
 }
diff --git a/pkg/tests/service/logservice.go b/pkg/tests/service/logservice.go
index e3e941552..65d89dcd4 100644
--- a/pkg/tests/service/logservice.go
+++ b/pkg/tests/service/logservice.go
@@ -17,7 +17,6 @@ package service
 import (
 	"path"
 	"strconv"
-	"strings"
 	"time"
 
 	"github.com/google/uuid"
@@ -30,10 +29,6 @@ import (
 	logpb "github.com/matrixorigin/matrixone/pkg/pb/logservice"
 )
 
-const (
-	tomlSeparator = ";"
-)
-
 var (
 	defaultDeploymentID         uint64 = 1
 	defaultRTTMillisecond       uint64 = 5
@@ -81,11 +76,11 @@ func buildLogConfig(index int, opt Options, address serviceAddress) logservice.C
 		ServiceAddress:      address.getLogListenAddress(index), // hakeeper client use this address
 		RaftAddress:         address.getLogRaftAddress(index),
 		GossipAddress:       address.getLogGossipAddress(index),
-		GossipSeedAddresses: strings.Join(address.getLogGossipSeedAddresses(), tomlSeparator),
+		GossipSeedAddresses: address.getLogGossipSeedAddresses(),
 	}
 	cfg.HeartbeatInterval.Duration = defaultHeartbeatInterval
 	cfg.HAKeeperTickInterval.Duration = defaultHAKeeperTickInterval
-	cfg.HAKeeperClientConfig.ServiceAddresses = strings.Join(address.listHAKeeperListenAddresses(), tomlSeparator)
+	cfg.HAKeeperClientConfig.ServiceAddresses = address.listHAKeeperListenAddresses()
 	return cfg
 }