Repository: coinbase/rosetta-cli
Pull request #341 (2022-08-09 00:53:21)

[cli]enable allinmemory in cli
### Motivation
speed up validation
enable customized memory usage conifg 

### Solution
this pr

### Open questions
<!--
(optional) Any open questions or feedback on design desired?
-->



Patch:
diff --git a/cmd/root.go b/cmd/root.go
index 45984c1..a6b6cb6 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -55,6 +55,8 @@ var (
 	dataResultFile         string
 	constructionResultFile string
 	dataDirectory          string
+	inMemoryMode           bool
+	tableSize              int64
 
 	// Config is the populated *configuration.Configuration from
 	// the configurationFile. If none is provided, this is set
@@ -263,6 +265,21 @@ default values.`,
 		"",
 		"Data-dir configures the location of logs and data for validation. This will override the data_directory from configuration file",
 	)
+
+	checkDataCmd.Flags().Int64Var(
+		&tableSize,
+		"table-size",
+		0,
+		"Table-size configures the TableSize for badger DB. If table-size != 0, this will override the table_size from configuration file",
+	)
+
+	checkDataCmd.Flags().BoolVar(
+		&inMemoryMode,
+		"in-memory-mode",
+		false,
+		"in-memory-mode configures badger DB inMeomry option. Only when in-memory-mode=true, this will override the all_in_memory_enabled",
+	)
+
 	rootCmd.AddCommand(checkDataCmd)
 	checkConstructionCmd.Flags().StringVar(
 		&asserterConfigurationFile,
@@ -372,6 +389,14 @@ func initConfig() {
 	if len(dataDirectory) != 0 {
 		Config.DataDirectory = dataDirectory
 	}
+
+	if inMemoryMode {
+		Config.AllInMemoryEnabled = inMemoryMode
+	}
+
+	if tableSize != 0 {
+		Config.TableSize = &tableSize
+	}
 }
 
 func ensureDataDirectoryExists() {
diff --git a/configuration/types.go b/configuration/types.go
index 603afbe..2534cbc 100644
--- a/configuration/types.go
+++ b/configuration/types.go
@@ -414,6 +414,21 @@ type Configuration struct {
 	// but can use 10s of GBs of RAM, even with pruning enabled.
 	MemoryLimitDisabled bool `json:"memory_limit_disabled"`
 
+	// AllInMemoryDisabled configures storage to increase memory
+	// usage. Enabling this massively increases performance
+	// but can use >20s of GBs of RAM, even with pruning enabled.
+	AllInMemoryEnabled bool `json:"all_in_memory_enabled"`
+
+	// TableSize unit is GB, enable users to define MaxTableSize
+	// when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config
+	// default value is 6, modification range is [2, 100]
+	TableSize *int64 `json:"table_size,omitempty"`
+
+	// TableSize unit is MB, enable users to define ValueLogFileSize
+	// when AllInMemoryEnabled == true or MemoryLimitDisabled== true, Cli will look up this config
+	// default value is 512, modification range is [128, 2048]
+	ValueLogFileSize *int64 `json:"value_log_file_size,omitempty"`
+
 	// SeenBlockWorkers is the number of goroutines spawned to store
 	// seen blocks in storage before we attempt to sequence. If not populated,
 	// this value defaults to runtime.NumCPU().
diff --git a/examples/configuration/default.json b/examples/configuration/default.json
index df5c68a..c2ca45a 100644
--- a/examples/configuration/default.json
+++ b/examples/configuration/default.json
@@ -15,6 +15,7 @@
  "log_configuration": false,
  "compression_disabled": false,
  "memory_limit_disabled": false,
+ "all_in_memory_enabled": false,
  "error_stack_trace_disabled": false,
  "coin_supported": false,
  "construction": null,
diff --git a/go.mod b/go.mod
index 4db9cbf..dcb81ec 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli
 go 1.16
 
 require (
-	github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4
+	github.com/coinbase/rosetta-sdk-go v0.7.11
 	github.com/fatih/color v1.13.0
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/go-cmp v0.5.6 // indirect
diff --git a/go.sum b/go.sum
index 8797de3..4db8ed0 100644
--- a/go.sum
+++ b/go.sum
@@ -97,8 +97,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk
 github.com/cloudflare/cloudflare-go v0.14.0/go.mod h1:EnwdgGMaFOruiPZRFSgn+TsQ3hQ7C/YWzIGLeu5c304=
 github.com/coinbase/kryptology v1.8.0 h1:Aoq4gdTsJhSU3lNWsD5BWmFSz2pE0GlmrljaOxepdYY=
 github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI=
-github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4 h1:cHRCEzEk+4xerPtuJgO5a71qp7jIwGDKCKq+pte2VHA=
-github.com/coinbase/rosetta-sdk-go v0.7.11-0.20220629212620-136b591fb3f4/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM=
+github.com/coinbase/rosetta-sdk-go v0.7.11 h1:T7camDGstlX2ENVE0QHR1AVu3asSXED0vg1xDaF6bYE=
+github.com/coinbase/rosetta-sdk-go v0.7.11/go.mod h1:HLTqSTSnOGLWHGTxoUJQO2TLuKkas1B9i/7ByerK6lM=
 github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ=
 github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
 github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q=


Test patch:
diff --git a/pkg/tester/data.go b/pkg/tester/data.go
index e74e9bd..12ed5a9 100644
--- a/pkg/tester/data.go
+++ b/pkg/tester/data.go
@@ -65,6 +65,18 @@ const (
 	// EndAtTipCheckInterval is the frequency that EndAtTip condition
 	// is evaludated
 	EndAtTipCheckInterval = 10 * time.Second
+	
+	//MinTableSize unit is GB
+	MinTableSize = int64(2)
+	
+	//MaxTableSize unit is GB
+	MaxTableSize = int64(100)
+
+	//MinTableSize unit is MB
+	MinValueLogFileSize = int64(128)
+	
+	//MaxTableSize unit is MB
+	MaxValueLogFileSize = int64(2048)
 )
 
 var _ http.Handler = (*DataTester)(nil)
@@ -152,17 +164,52 @@ func InitializeData(
 	}
 
 	opts := []database.BadgerOption{}
-	if config.CompressionDisabled {
-		opts = append(opts, database.WithoutCompression())
-	}
-	if config.MemoryLimitDisabled {
+	dataPathBackup := dataPath
+
+	if config.AllInMemoryEnabled{
 		opts = append(
 			opts,
-			database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)),
+			database.WithCustomSettings(database.AllInMemoryBadgerOptions(dataPath)),
+			database.WithoutCompression(),
 		)
+		// for all in memory mode, the path need to be "", as badgerDB will not write to disk
+		dataPathBackup = ""
+	} else {
+		if config.CompressionDisabled {
+			opts = append(opts, database.WithoutCompression())
+		}
+		if config.MemoryLimitDisabled {
+			opts = append(
+				opts,
+				database.WithCustomSettings(database.PerformanceBadgerOptions(dataPath)),
+			)
+		}
+	}
+
+	// If we enable all-in-memory or L0-in-memory mode, badger DB's TableSize and ValueLogFileSize will change
+	// according to users config. tableSize means the LSM table size, when the table more than the tableSize, 
+	// will trigger a compact. 
+	// In default mode, we will not change the badger DB's TableSize and ValueLogFileSize for limiting memory usage
+	if config.AllInMemoryEnabled || config.MemoryLimitDisabled {
+		if(config.TableSize != nil) {
+			if(*config.TableSize >= MinTableSize && *config.TableSize <= MaxTableSize) {
+				opts = append(
+					opts,
+					database.WithTableSize(*config.TableSize),
+				)
+			}
+		}
+		if(config.ValueLogFileSize != nil) {
+			if(*config.TableSize >= MinValueLogFileSize && *config.TableSize <= MinValueLogFileSize) {
+				opts = append(
+					opts,
+					database.WithValueLogFileSize(*config.TableSize),
+				)
+			}
+		}
 	}
 
-	localStore, err := database.NewBadgerDatabase(ctx, dataPath, opts...)
+	localStore, err := database.NewBadgerDatabase(ctx, dataPathBackup, opts...)
 	if err != nil {
 		return nil, fmt.Errorf("%s: unable to initialize database", err.Error())
 	}