Repository: matrixorigin/matrixone
Pull request #4521 (2022-08-15 15:42:55)

logservice: made tan's buffer size configurable
## What type of PR is this?

- [ ] API-change
- [ ] BUG
- [ ] Improvement
- [ ] Documentation
- [ ] Feature
- [x] Test and CI
- [ ] Code Refactoring

## Which issue(s) this PR fixes:

issue #

## What this PR does / why we need it:

Made tan's buffer size configurable, reduced buffer size in tests to be 16KBytes each, it was 4MBytes each. Also set debug.SetMemoryLimit(1GBytes) to further reduce memory usage when running TestGossipInSimulatedCluster.


Patch:
diff --git a/go.mod b/go.mod
index 84a42caeb..4eef28222 100644
--- a/go.mod
+++ b/go.mod
@@ -21,7 +21,7 @@ require (
 	github.com/google/gofuzz v1.2.0
 	github.com/google/gops v0.3.25
 	github.com/google/uuid v1.3.0
-	github.com/lni/dragonboat/v4 v4.0.0-20220803152440-a83f853de8b1
+	github.com/lni/dragonboat/v4 v4.0.0-20220815145555-6f622e8bcbef
 	github.com/lni/goutils v1.3.1-0.20220604063047-388d67b4dbc4
 	github.com/matrixorigin/matrixcube v0.3.1-0.20220606032431-c944d801f1e5
 	github.com/matrixorigin/simdcsv v0.0.0-20220815110641-d4a2c7888f52
diff --git a/go.sum b/go.sum
index 72a03bb56..f51b47258 100644
--- a/go.sum
+++ b/go.sum
@@ -377,8 +377,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvfxNnFqi74g=
 github.com/labstack/echo/v4 v4.5.0/go.mod h1:czIriw4a0C1dFun+ObrXp7ok03xON0N1awStJ6ArI7Y=
 github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k=
-github.com/lni/dragonboat/v4 v4.0.0-20220803152440-a83f853de8b1 h1:ds2JUqJxw8RtmwOwPJBNB2RZF+AfDFqtUgfa5efqxjw=
-github.com/lni/dragonboat/v4 v4.0.0-20220803152440-a83f853de8b1/go.mod h1:bR6ZGoUwApH4/P4+AezNGT0xehljg+j+Z/Q2Fz+Y4a0=
+github.com/lni/dragonboat/v4 v4.0.0-20220815145555-6f622e8bcbef h1:Pk7v7YAMqQrL0nbHyoffdmr9BArHN03hEqBgeru8VcI=
+github.com/lni/dragonboat/v4 v4.0.0-20220815145555-6f622e8bcbef/go.mod h1:bR6ZGoUwApH4/P4+AezNGT0xehljg+j+Z/Q2Fz+Y4a0=
 github.com/lni/goutils v1.3.0/go.mod h1:PUPtBAnZlRPUKWUXCsYkIRWubJbtNHpTAee0sczhlf4=
 github.com/lni/goutils v1.3.1-0.20220604063047-388d67b4dbc4 h1:6gzI38ZJmbzZ7oZebXz6jII0uVK+MZ3+ds+7mIt1ioI=
 github.com/lni/goutils v1.3.1-0.20220604063047-388d67b4dbc4/go.mod h1:LIHvF0fflR+zyXUQFQOiHPpKANf3UIr7DFIv5CBPOoU=
diff --git a/pkg/logservice/config.go b/pkg/logservice/config.go
index d9dfb13fe..38d502ddd 100644
--- a/pkg/logservice/config.go
+++ b/pkg/logservice/config.go
@@ -34,6 +34,10 @@ const (
 	defaultServiceAddress = "0.0.0.0:32000"
 	defaultRaftAddress    = "0.0.0.0:32001"
 	defaultGossipAddress  = "0.0.0.0:32002"
+
+	defaultGossipProbeInterval = 50 * time.Millisecond
+	defaultHeartbeatInterval   = time.Second
+	defaultLogDBBufferSize     = 768 * 1024
 )
 
 var (
@@ -41,11 +45,6 @@ var (
 	ErrInvalidConfig          = moerr.NewError(moerr.BAD_CONFIGURATION, "invalid log configuration")
 )
 
-const (
-	defaultGossipProbeInterval = 50 * time.Millisecond
-	defaultHeartbeatInterval   = time.Second
-)
-
 // Config defines the Configurations supported by the Log Service.
 type Config struct {
 	// FS is the underlying virtual FS used by the log service. Leave it as empty
@@ -72,6 +71,8 @@ type Config struct {
 	RaftAddress string `toml:"raft-address"`
 	// RaftListenAddress is the local listen address of the RaftAddress.
 	RaftListenAddress string `toml:"raft-listen-address"`
+	// LogDBBufferSize is the size of the logdb buffer in bytes.
+	LogDBBufferSize uint64 `toml:"logdb-buffer-size"`
 	// GossipAddress is the address used for accepting gossip communication.
 	GossipAddress string `toml:"gossip-address"`
 	// GossipListenAddress is the local listen address of the GossipAddress
@@ -230,6 +231,9 @@ func (c *Config) Validate() error {
 	if len(c.RaftAddress) == 0 && len(c.RaftListenAddress) != 0 {
 		return errors.Wrapf(ErrInvalidConfig, "RaftAddress not set")
 	}
+	if c.LogDBBufferSize == 0 {
+		return errors.Wrapf(ErrInvalidConfig, "LogDBBufferSize not set")
+	}
 	if len(c.GossipAddress) == 0 && len(c.GossipListenAddress) != 0 {
 		return errors.Wrapf(ErrInvalidConfig, "GossipAddress not set")
 	}
@@ -299,6 +303,9 @@ func (c *Config) Fill() {
 	} else if len(c.RaftAddress) != 0 && len(c.RaftListenAddress) == 0 {
 		c.RaftListenAddress = c.RaftAddress
 	}
+	if c.LogDBBufferSize == 0 {
+		c.LogDBBufferSize = defaultLogDBBufferSize
+	}
 	if len(c.GossipAddress) == 0 {
 		c.GossipAddress = defaultGossipAddress
 		c.GossipListenAddress = defaultGossipAddress
diff --git a/pkg/logservice/store.go b/pkg/logservice/store.go
index 9baeadcb3..781e7e371 100644
--- a/pkg/logservice/store.go
+++ b/pkg/logservice/store.go
@@ -72,6 +72,8 @@ func getNodeHostConfig(cfg Config) config.NodeHostConfig {
 	if cfg.GossipProbeInterval.Duration == 0 {
 		panic("cfg.GossipProbeInterval.Duration is 0")
 	}
+	logdb := config.GetTinyMemLogDBConfig()
+	logdb.KVWriteBufferSize = cfg.LogDBBufferSize
 	return config.NodeHostConfig{
 		DeploymentID:        cfg.DeploymentID,
 		NodeHostID:          cfg.UUID,
@@ -86,6 +88,7 @@ func getNodeHostConfig(cfg Config) config.NodeHostConfig {
 			// FIXME: dragonboat need to be updated to make this field a first class
 			// citizen
 			TestGossipProbeInterval: cfg.GossipProbeInterval.Duration,
+			LogDB:                   logdb,
 		},
 		Gossip: config.GossipConfig{
 			BindAddress:      cfg.GossipListenAddress,


Test patch:
diff --git a/pkg/logservice/service_test.go b/pkg/logservice/service_test.go
index 1ea21392b..c5f52dcc0 100644
--- a/pkg/logservice/service_test.go
+++ b/pkg/logservice/service_test.go
@@ -17,6 +17,7 @@ package logservice
 import (
 	"context"
 	"fmt"
+	"runtime/debug"
 	"sync"
 	"testing"
 	"time"
@@ -595,6 +596,7 @@ func TestShardInfoCanBeQueried(t *testing.T) {
 
 func TestGossipInSimulatedCluster(t *testing.T) {
 	defer leaktest.AfterTest(t)()
+	debug.SetMemoryLimit(1 << 30)
 	// start all services
 	nodeCount := 24
 	shardCount := nodeCount / 3
@@ -622,7 +624,8 @@ func TestGossipInSimulatedCluster(t *testing.T) {
 				"127.0.0.1:6082",
 				"127.0.0.1:6092",
 			},
-			DisableWorkers: true,
+			DisableWorkers:  true,
+			LogDBBufferSize: 1024 * 16,
 		}
 		cfg.GossipProbeInterval.Duration = 350 * time.Millisecond
 		configs = append(configs, cfg)