Repository: zeromicro/go-zero
Pull request #2767 (2023-01-08 14:28:35)

feat: add config to truncate long log content

codecov-commenter: # [Codecov](https://codecov.io/gh/zeromicro/go-zero/pull/2767?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) Report
> Merging [#2767](https://codecov.io/gh/zeromicro/go-zero/pull/2767?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) (2807d3a) into [master](https://codecov.io/gh/zeromicro/go-zero/commit/26c541b9cbd906af138795db349eebcca6a0dc8e?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) (26c541b) will **increase** coverage by `0.02%`.
> The diff coverage is `100.00%`.


| [Impacted Files](https://codecov.io/gh/zeromicro/go-zero/pull/2767?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None) | Coverage Δ | |
|---|---|---|
| [core/logx/logs.go](https://codecov.io/gh/zeromicro/go-zero/pull/2767?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#diff-Y29yZS9sb2d4L2xvZ3MuZ28=) | `95.86% <100.00%> (+0.03%)` | :arrow_up: |
| [core/logx/writer.go](https://codecov.io/gh/zeromicro/go-zero/pull/2767?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#diff-Y29yZS9sb2d4L3dyaXRlci5nbw==) | `94.40% <100.00%> (+0.15%)` | :arrow_up: |
| [zrpc/client.go](https://codecov.io/gh/zeromicro/go-zero/pull/2767?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#diff-enJwYy9jbGllbnQuZ28=) | `93.87% <100.00%> (+0.12%)` | :arrow_up: |
| [core/logx/rotatelogger.go](https://codecov.io/gh/zeromicro/go-zero/pull/2767?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=None#diff-Y29yZS9sb2d4L3JvdGF0ZWxvZ2dlci5nbw==) | `72.69% <0.00%> (+1.10%)` | :arrow_up: |


Patch:
diff --git a/core/logx/config.go b/core/logx/config.go
index 7d1a4c1e..8f0398fd 100644
--- a/core/logx/config.go
+++ b/core/logx/config.go
@@ -8,6 +8,7 @@ type LogConf struct {
 	TimeFormat          string `json:",optional"`
 	Path                string `json:",default=logs"`
 	Level               string `json:",default=info,options=[debug,info,error,severe]"`
+	MaxContentLength    uint32 `json:",optional"`
 	Compress            bool   `json:",optional"`
 	Stat                bool   `json:",default=true"`
 	KeepDays            int    `json:",optional"`
diff --git a/core/logx/logs.go b/core/logx/logs.go
index 65047d49..734629a2 100644
--- a/core/logx/logs.go
+++ b/core/logx/logs.go
@@ -20,6 +20,8 @@ var (
 	timeFormat = "2006-01-02T15:04:05.000Z07:00"
 	logLevel   uint32
 	encoding   uint32 = jsonEncodingType
+	// maxContentLength is used to truncate the log content, 0 for not truncating.
+	maxContentLength uint32
 	// use uint32 for atomic operations
 	disableLog  uint32
 	disableStat uint32
@@ -238,6 +240,8 @@ func SetUp(c LogConf) (err error) {
 			timeFormat = c.TimeFormat
 		}
 
+		atomic.StoreUint32(&maxContentLength, c.MaxContentLength)
+
 		switch c.Encoding {
 		case plainEncoding:
 			atomic.StoreUint32(&encoding, plainEncodingType)
diff --git a/core/logx/vars.go b/core/logx/vars.go
index b95c3215..441bf973 100644
--- a/core/logx/vars.go
+++ b/core/logx/vars.go
@@ -16,13 +16,13 @@ const (
 const (
 	jsonEncodingType = iota
 	plainEncodingType
+)
 
+const (
 	plainEncoding    = "plain"
 	plainEncodingSep = '\t'
 	sizeRotationRule = "size"
-)
 
-const (
 	accessFilename = "access.log"
 	errorFilename  = "error.log"
 	severeFilename = "severe.log"
@@ -53,6 +53,7 @@ const (
 	spanKey      = "span"
 	timestampKey = "@timestamp"
 	traceKey     = "trace"
+	truncatedKey = "truncated"
 )
 
 var (
@@ -60,4 +61,6 @@ var (
 	ErrLogPathNotSet = errors.New("log path must be set")
 	// ErrLogServiceNameNotSet is an error that indicates that the service name is not set.
 	ErrLogServiceNameNotSet = errors.New("log service name must be set")
+
+	truncatedField = Field(truncatedKey, true)
 )
diff --git a/core/logx/writer.go b/core/logx/writer.go
index 2d4ade04..d9c2fb00 100644
--- a/core/logx/writer.go
+++ b/core/logx/writer.go
@@ -278,6 +278,15 @@ func combineGlobalFields(fields []LogField) []LogField {
 }
 
 func output(writer io.Writer, level string, val interface{}, fields ...LogField) {
+	// only truncate string content, don't know how to truncate the values of other types.
+	if v, ok := val.(string); ok {
+		maxLen := atomic.LoadUint32(&maxContentLength)
+		if maxLen > 0 && len(v) > int(maxLen) {
+			val = v[:maxLen]
+			fields = append(fields, truncatedField)
+		}
+	}
+
 	fields = combineGlobalFields(fields)
 
 	switch atomic.LoadUint32(&encoding) {
diff --git a/core/stores/mon/bulkinserter.go b/core/stores/mon/bulkinserter.go
index 74cbb7d9..7ef15ca5 100644
--- a/core/stores/mon/bulkinserter.go
+++ b/core/stores/mon/bulkinserter.go
@@ -25,25 +25,15 @@ type (
 	}
 )
 
-// Deprecated. Use NewBatchInserter instead.
-func NewBulkInserter(coll *mongo.Collection, interval ...time.Duration) *BulkInserter {
-	return newBulkInserter(coll, interval...)
-}
-
-// NewBatchInserter returns a BulkInserter.
-func NewBatchInserter(coll Collection, interval ...time.Duration) (*BulkInserter, error) {
+// NewBulkInserter returns a BulkInserter.
+func NewBulkInserter(coll Collection, interval ...time.Duration) (*BulkInserter, error) {
 	cloneColl, err := coll.Clone()
 	if err != nil {
 		return nil, err
 	}
 
-	return newBulkInserter(cloneColl, interval...), nil
-}
-
-// newBulkInserter returns a BulkInserter.
-func newBulkInserter(coll *mongo.Collection, interval ...time.Duration) *BulkInserter {
 	inserter := &dbInserter{
-		collection: coll,
+		collection: cloneColl,
 	}
 
 	duration := flushInterval
@@ -54,7 +44,7 @@ func newBulkInserter(coll *mongo.Collection, interval ...time.Duration) *BulkIns
 	return &BulkInserter{
 		executor: executors.NewPeriodicalExecutor(duration, inserter),
 		inserter: inserter,
-	}
+	}, nil
 }
 
 // Flush flushes the inserter, writes all pending records.
diff --git a/zrpc/client.go b/zrpc/client.go
index 790a5a75..9966d038 100644
--- a/zrpc/client.go
+++ b/zrpc/client.go
@@ -89,6 +89,7 @@ func NewClientWithTarget(target string, opts ...ClientOption) (Client, error) {
 		Breaker:    true,
 		Timeout:    true,
 	}
+
 	return internal.NewClient(target, middlewares, opts...)
 }
 


Test patch:
diff --git a/core/logx/writer_test.go b/core/logx/writer_test.go
index 9f9c3598..c2b476e4 100644
--- a/core/logx/writer_test.go
+++ b/core/logx/writer_test.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"log"
+	"sync/atomic"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -157,9 +158,40 @@ func TestWritePlainAny(t *testing.T) {
 
 }
 
+func TestLogWithLimitContentLength(t *testing.T) {
+	maxLen := atomic.LoadUint32(&maxContentLength)
+	atomic.StoreUint32(&maxContentLength, 10)
+
+	t.Cleanup(func() {
+		atomic.StoreUint32(&maxContentLength, maxLen)
+	})
+
+	t.Run("alert", func(t *testing.T) {
+		var buf bytes.Buffer
+		w := NewWriter(&buf)
+		w.Info("1234567890")
+		var v1 mockedEntry
+		if err := json.Unmarshal(buf.Bytes(), &v1); err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, "1234567890", v1.Content)
+		assert.False(t, v1.Truncated)
+
+		buf.Reset()
+		var v2 mockedEntry
+		w.Info("12345678901")
+		if err := json.Unmarshal(buf.Bytes(), &v2); err != nil {
+			t.Fatal(err)
+		}
+		assert.Equal(t, "1234567890", v2.Content)
+		assert.True(t, v2.Truncated)
+	})
+}
+
 type mockedEntry struct {
-	Level   string `json:"level"`
-	Content string `json:"content"`
+	Level     string `json:"level"`
+	Content   string `json:"content"`
+	Truncated bool   `json:"truncated"`
 }
 
 type easyToCloseWriter struct{}
diff --git a/core/stores/mon/bulkinserter_test.go b/core/stores/mon/bulkinserter_test.go
index f2889d35..ebd0ec4a 100644
--- a/core/stores/mon/bulkinserter_test.go
+++ b/core/stores/mon/bulkinserter_test.go
@@ -15,24 +15,7 @@ func TestBulkInserter(t *testing.T) {
 
 	mt.Run("test", func(mt *mtest.T) {
 		mt.AddMockResponses(mtest.CreateSuccessResponse(bson.D{{Key: "ok", Value: 1}}...))
-		bulk := NewBulkInserter(mt.Coll)
-		bulk.SetResultHandler(func(result *mongo.InsertManyResult, err error) {
-			assert.Nil(t, err)
-			assert.Equal(t, 2, len(result.InsertedIDs))
-		})
-		bulk.Insert(bson.D{{Key: "foo", Value: "bar"}})
-		bulk.Insert(bson.D{{Key: "foo", Value: "baz"}})
-		bulk.Flush()
-	})
-}
-
-func TestBatchInserter(t *testing.T) {
-	mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
-	defer mt.Close()
-
-	mt.Run("test", func(mt *mtest.T) {
-		mt.AddMockResponses(mtest.CreateSuccessResponse(bson.D{{Key: "ok", Value: 1}}...))
-		bulk, err := NewBatchInserter(createModel(mt).Collection)
+		bulk, err := NewBulkInserter(createModel(mt).Collection)
 		assert.Equal(t, err, nil)
 		bulk.SetResultHandler(func(result *mongo.InsertManyResult, err error) {
 			assert.Nil(t, err)