Repository: coinbase/rosetta-cli
Pull request #376 (2022-12-08 08:22:21)

enable requestUUID
Fixes # .
use two methods to add requestUUID
1, add it into ctx
2, add it to logger

<img width="1459" alt="Screenshot 2022-12-08 at 12 17 56 AM" src="https://user-images.githubusercontent.com/96205264/206395729-f7e43594-1533-4bda-8298-67c7b0b19fb8.png">

<img width="1460" alt="Screenshot 2022-12-08 at 12 19 26 AM" src="https://user-images.githubusercontent.com/96205264/206395756-9769feeb-0792-4b19-b035-13ec81466441.png">

### Motivation
<!--
Does this solve a bug? Enable a new use-case? Improve an existing behavior? Concrete examples are helpful here.
-->

### Solution
<!--
What is the solution here from a high level. What are the key technical decisions and why were they made?
-->

### Open questions
<!--
(optional) Any open questions or feedback on design desired?
-->



Patch:
diff --git a/cmd/check_construction.go b/cmd/check_construction.go
index 497497d..c05c674 100644
--- a/cmd/check_construction.go
+++ b/cmd/check_construction.go
@@ -20,6 +20,7 @@ import (
 	"time"
 
 	cliErrs "github.com/coinbase/rosetta-cli/pkg/errors"
+	"github.com/coinbase/rosetta-cli/pkg/logger"
 
 	"github.com/coinbase/rosetta-cli/pkg/results"
 	"github.com/coinbase/rosetta-cli/pkg/tester"
@@ -147,6 +148,8 @@ func runCheckConstructionCmd(_ *cobra.Command, _ []string) error {
 	}
 
 	g, ctx := errgroup.WithContext(ctx)
+	ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID)
+
 	g.Go(func() error {
 		return constructionTester.StartPeriodicLogger(ctx)
 	})
diff --git a/cmd/check_data.go b/cmd/check_data.go
index 962f364..8163765 100644
--- a/cmd/check_data.go
+++ b/cmd/check_data.go
@@ -19,6 +19,7 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/coinbase/rosetta-cli/pkg/logger"
 	"github.com/coinbase/rosetta-cli/pkg/results"
 	"github.com/coinbase/rosetta-cli/pkg/tester"
 	"github.com/coinbase/rosetta-sdk-go/fetcher"
@@ -154,6 +155,8 @@ func runCheckDataCmd(_ *cobra.Command, _ []string) error {
 	defer dataTester.CloseDatabase(ctx)
 
 	g, ctx := errgroup.WithContext(ctx)
+	ctx = logger.AddRequestUUIDToContext(ctx, Config.RequestUUID)
+	
 	g.Go(func() error {
 		return dataTester.StartPeriodicLogger(ctx)
 	})
diff --git a/cmd/root.go b/cmd/root.go
index 0b36a2f..6c9c3fc 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -57,6 +57,7 @@ var (
 	dataDirectory          string
 	inMemoryMode           bool
 	tableSize              int64
+	requestUUID            string
 
 	// Config is the populated *configuration.Configuration from
 	// the configurationFile. If none is provided, this is set
@@ -280,6 +281,13 @@ default values.`,
 		"In-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true, this will override the all_in_memory_enabled",
 	)
 
+	checkDataCmd.Flags().StringVar(
+		&requestUUID,
+		"requestUUID",
+		"",
+		"requestUUID configures the requestUUID in logs, which aims to enable search logs by requestUUID",
+	)
+
 	rootCmd.AddCommand(checkDataCmd)
 	checkConstructionCmd.Flags().StringVar(
 		&asserterConfigurationFile,
@@ -308,6 +316,14 @@ default values.`,
 		"",
 		"Result-file configures the location of validation result. This will override the results_output_file from configuration file",
 	)
+
+	checkConstructionCmd.Flags().StringVar(
+		&requestUUID,
+		"requestUUID",
+		"",
+		"requestUUID configures the requestUUID in logs, which aims to enable search logs by requestUUID",
+	)
+
 	rootCmd.AddCommand(checkConstructionCmd)
 
 	// View Commands
@@ -399,6 +415,10 @@ func initConfig() {
 	} else if tableSize != -1 {
 		log.Fatalf("table-size %d is not in the range [2, 100], please check your input", tableSize)
 	}
+
+	if len(requestUUID) != 0 {
+		Config.RequestUUID = requestUUID
+	}
 }
 
 func ensureDataDirectoryExists() {
diff --git a/configuration/types.go b/configuration/types.go
index 3de41ac..b7b5d7b 100644
--- a/configuration/types.go
+++ b/configuration/types.go
@@ -367,6 +367,9 @@ type Configuration struct {
 	// binary is being executed.
 	DataDirectory string `json:"data_directory"`
 
+	// Make search log easier when validation pool is working on different request
+	RequestUUID string `json:"requestUUID,omitempty"`
+
 	// HTTPTimeout is the timeout for a HTTP request in seconds.
 	HTTPTimeout uint64 `json:"http_timeout"`
 
diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go
index c6f99ba..72e7904 100644
--- a/pkg/logger/logger.go
+++ b/pkg/logger/logger.go
@@ -38,7 +38,11 @@ var _ statefulsyncer.Logger = (*Logger)(nil)
 
 type CheckType string
 
+type contextKey int
+
 const (
+	RequestUUID contextKey = iota
+
 	// blockStreamFile contains the stream of processed
 	// blocks and whether they were added or removed.
 	blockStreamFile = "blocks.txt"
@@ -78,6 +82,7 @@ type Logger struct {
 	logTransactions   bool
 	logBalanceChanges bool
 	logReconciliation bool
+	logRequestUUID    string
 
 	lastStatsMessage    string
 	lastProgressMessage string
@@ -94,9 +99,10 @@ func NewLogger(
 	logReconciliation bool,
 	checkType CheckType,
 	network *types.NetworkIdentifier,
+	logRequestUUID string,
 	fields ...zap.Field,
 ) (*Logger, error) {
-	zapLogger, err := buildZapLogger(checkType, network, fields...)
+	zapLogger, err := buildZapLogger(checkType, network, logRequestUUID, fields...)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build zap logger: %w", err)
 	}
@@ -106,6 +112,7 @@ func NewLogger(
 		logTransactions:   logTransactions,
 		logBalanceChanges: logBalanceChanges,
 		logReconciliation: logReconciliation,
+		logRequestUUID:    logRequestUUID,
 		zapLogger:         zapLogger,
 	}, nil
 }
@@ -113,14 +120,16 @@ func NewLogger(
 func buildZapLogger(
 	checkType CheckType,
 	network *types.NetworkIdentifier,
+	requestUUID string,
 	fields ...zap.Field,
 ) (*zap.Logger, error) {
-	config := zap.NewDevelopmentConfig()
+	config := zap.NewProductionConfig()
 	config.EncoderConfig.EncodeLevel = zapcore.CapitalColorLevelEncoder
 
 	baseSlice := []zap.Field{
 		zap.String("blockchain", network.Blockchain),
 		zap.String("network", network.Network),
+		zap.String("requestUUID", requestUUID),
 		zap.String("check_type", string(checkType)),
 	}
 	mergedSlice := append(baseSlice, fields...)
@@ -151,6 +160,8 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta
 		status.Stats.ReconciliationCoverage*utils.OneHundred,
 	)
 
+	statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID)
+
 	// Don't print out the same stats message twice.
 	if statsMessage == l.lastStatsMessage {
 		return
@@ -175,6 +186,8 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta
 		status.Progress.ReconcilerLastIndex,
 	)
 
+	progressMessage = AddRequestUUID(progressMessage, l.logRequestUUID)
+
 	// Don't print out the same progress message twice.
 	if progressMessage == l.lastProgressMessage {
 		return
@@ -202,6 +215,8 @@ func (l *Logger) LogConstructionStatus(
 		return
 	}
 
+	statsMessage = AddRequestUUID(statsMessage, l.logRequestUUID)
+
 	l.lastStatsMessage = statsMessage
 	color.Cyan(statsMessage)
 }
@@ -216,7 +231,7 @@ func LogMemoryStats(ctx context.Context) {
 		memUsage.System,
 		memUsage.GarbageCollections,
 	)
-
+	statsMessage = AddRequestUUIDFromContext(ctx, statsMessage)
 	color.Cyan(statsMessage)
 }
 
@@ -242,12 +257,13 @@ func (l *Logger) AddBlockStream(
 	defer closeFile(f)
 
 	blockString := fmt.Sprintf(
-		"%s Block %d:%s with Parent Block %d:%s\n",
+		"%s Block %d:%s with Parent Block %d:%s, RequestUUID: %s\n",
 		addEvent,
 		block.BlockIdentifier.Index,
 		block.BlockIdentifier.Hash,
 		block.ParentBlockIdentifier.Index,
 		block.ParentBlockIdentifier.Hash,
+		l.logRequestUUID,
 	)
 	fmt.Print(blockString)
 	if _, err := f.WriteString(blockString); err != nil {
@@ -279,10 +295,11 @@ func (l *Logger) RemoveBlockStream(
 	defer closeFile(f)
 
 	blockString := fmt.Sprintf(
-		"%s Block %d:%s\n",
+		"%s Block %d:%s, RequestUUID: %s\n",
 		removeEvent,
 		block.Index,
 		block.Hash,
+		l.logRequestUUID,
 	)
 	fmt.Print(blockString)
 	_, err = f.WriteString(blockString)
@@ -316,10 +333,11 @@ func (l *Logger) TransactionStream(
 
 	for _, tx := range block.Transactions {
 		transactionString := fmt.Sprintf(
-			"Transaction %s at Block %d:%s\n",
+			"Transaction %s at Block %d:%s, RequestUUID: %s\n",
 			tx.TransactionIdentifier.Hash,
 			block.BlockIdentifier.Index,
 			block.BlockIdentifier.Hash,
+			l.logRequestUUID,
 		)
 		fmt.Print(transactionString)
 		_, err = f.WriteString(transactionString)
@@ -394,7 +412,7 @@ func (l *Logger) BalanceStream(
 			balanceChange.Block.Index,
 			balanceChange.Block.Hash,
 		)
-
+		balanceLog = AddRequestUUID(balanceLog, l.logRequestUUID)
 		if _, err := f.WriteString(fmt.Sprintf("%s\n", balanceLog)); err != nil {
 			return fmt.Errorf("failed to write balance log %s: %w", balanceLog, err)
 		}
@@ -435,13 +453,14 @@ func (l *Logger) ReconcileSuccessStream(
 	)
 
 	reconciliationSuccessString := fmt.Sprintf(
-		"Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s\n",
+		"Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s, RequestUUID: %s\n",
 		reconciliationType,
 		types.AccountString(account),
 		types.CurrencyString(currency),
 		balance,
 		block.Index,
 		block.Hash,
+		l.logRequestUUID,
 	)
 	_, err = f.WriteString(reconciliationSuccessString)
 	if err != nil {
@@ -500,7 +519,7 @@ func (l *Logger) ReconcileFailureStream(
 	defer closeFile(f)
 
 	reconciliationFailureString := fmt.Sprintf(
-		"Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s\n",
+		"Type:%s Account: %s Currency: %s Block: %s:%d computed: %s live: %s, RequestUUID: %s\n",
 		reconciliationType,
 		types.AccountString(account),
 		types.CurrencyString(currency),
@@ -508,6 +527,7 @@ func (l *Logger) ReconcileFailureStream(
 		block.Index,
 		computedBalance,
 		liveBalance,
+		l.logRequestUUID,
 	)
 	_, err = f.WriteString(reconciliationFailureString)
 	if err != nil {
@@ -565,3 +585,35 @@ func LogTransactionCreated(
 		transactionIdentifier.Hash,
 	)
 }
+
+// Add requestUUID to the tip
+func AddRequestUUIDFromContext(ctx context.Context, msg string) string {
+	requestUUID := requestUUIDFromContext(ctx)
+	if requestUUID != "" {
+		msg = fmt.Sprintf("%s,RequestUUID: %s\n", msg, requestUUID)
+	}
+	return msg
+}
+
+// Add requestUUID to the tip
+func AddRequestUUID(msg string, requestUUID string) string {
+	if requestUUID != "" {
+		msg = fmt.Sprintf("%s,RequestUUID: %s\n", msg, requestUUID)
+	}
+	return msg
+}
+
+// AddRequestUUIDToContext will add a requestUUIDto the context, and return the new context
+func AddRequestUUIDToContext(ctx context.Context, uuid string) context.Context {
+	return context.WithValue(ctx, RequestUUID, uuid)
+}
+
+// requestUUIDFromContext is used to extract a request UUID from a context
+func requestUUIDFromContext(ctx context.Context) string {
+	switch v := ctx.Value(RequestUUID).(type) {
+	case string:
+		return v
+	default:
+		return ""
+	}
+}


Test patch:
diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go
index 90e6701..ba8a806 100644
--- a/pkg/tester/construction.go
+++ b/pkg/tester/construction.go
@@ -123,6 +123,7 @@ func InitializeConstruction(
 		false,
 		logger.Construction,
 		network,
+		config.RequestUUID,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("unable to initialize logger with error: %w", err)
diff --git a/pkg/tester/data.go b/pkg/tester/data.go
index d3d9f56..10fd665 100644
--- a/pkg/tester/data.go
+++ b/pkg/tester/data.go
@@ -78,6 +78,9 @@ const (
 
 	//MaxTableSize unit is MB
 	MaxValueLogFileSize = int64(2048)
+	
+	// empty requestUUID
+	EmptyRequestUUID = ""
 )
 
 var _ http.Handler = (*DataTester)(nil)
@@ -237,6 +240,7 @@ func InitializeData(
 		config.Data.LogReconciliations,
 		logger.Data,
 		network,
+		config.RequestUUID,
 	)
 	if err != nil {
 		return nil, fmt.Errorf("unable to initialize logger with error: %w", err)
@@ -345,12 +349,7 @@ func InitializeData(
 		//
 		// We need to do this after instantiating the balance storage handler
 		// because it is invoked within BootstrapBalances.
-		//
-		// We only need to bootstrap balances when we run this test from
-		// genesis block. If it is not genesis block, we use the balances from
-		// previous block
-		if (config.Data.StartIndex == nil || *config.Data.StartIndex == genesisBlock.Index) &&
-			len(config.Data.BootstrapBalances) > 0 {
+		if len(config.Data.BootstrapBalances) > 0 {
 			_, err := blockStorage.GetHeadBlockIdentifier(ctx)
 			switch {
 			case err == storageErrs.ErrHeadBlockNotFound:
@@ -1084,6 +1083,7 @@ func (t *DataTester) recursiveOpSearch(
 		false,
 		logger.Data,
 		t.network,
+		EmptyRequestUUID,
 	)
 
 	if err != nil {
diff --git a/pkg/tester/general.go b/pkg/tester/general.go
index 0edc1a3..776070c 100644
--- a/pkg/tester/general.go
+++ b/pkg/tester/general.go
@@ -28,9 +28,6 @@ const (
 	// MemoryLoggingFrequency is the frequency that memory
 	// usage stats are logged to the terminal.
 	MemoryLoggingFrequency = 10 * time.Second
-
-	// ReadHeaderTimeout is the header timeout for server
-	ReadHeaderTimeout = 5 * time.Second
 )
 
 // LogMemoryLoop runs a loop that logs memory usage.
@@ -60,9 +57,8 @@ func StartServer(
 	port uint,
 ) error {
 	server := &http.Server{
-		Addr:              fmt.Sprintf(":%d", port),
-		Handler:           handler,
-		ReadHeaderTimeout: ReadHeaderTimeout,
+		Addr:    fmt.Sprintf(":%d", port),
+		Handler: handler,
 	}
 
 	go func() {