Repository: coinbase/rosetta-cli
Pull request #171 (2020-10-22 16:51:50)

Optionally Disable Balance Pre-Fetch
This PR allows the caller to optionally disable balance pre-fetching. This can SIGNIFICANTLY improve `check:data` speed when syncing from genesis (as all newly seen accounts will have balance `0` before operations are applied).

### Changes
- [x] Update version
- [x] Add `initial_balance_fetch_disabled` to both data and construction config
- [x] use `initial_balance_fetch_disabled` in balance storage helper
- [x] Prevent this config if balance exemptions are allowed


Patch:
diff --git a/cmd/root.go b/cmd/root.go
index 92a2712..19d0bc8 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -247,6 +247,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print rosetta-cli version",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("v0.5.15")
+		fmt.Println("v0.5.16")
 	},
 }
diff --git a/configuration/types.go b/configuration/types.go
index 608cba6..2349f82 100644
--- a/configuration/types.go
+++ b/configuration/types.go
@@ -146,6 +146,16 @@ type ConstructionConfiguration struct {
 	// Quiet is a boolean indicating if all request and response
 	// logging should be silenced.
 	Quiet bool `json:"quiet,omitempty"`
+
+	// InitialBalanceFetchDisabled configures rosetta-cli
+	// not to lookup the balance of newly seen accounts at the
+	// parent block before applying operations. Disabling this
+	// is only a good idea if you create multiple new accounts each block
+	// and don't fund any accounts before starting check:construction.
+	//
+	// This is a separate config from the data config because it
+	// is usually false whereas the data config by the same name is usually true.
+	InitialBalanceFetchDisabled bool `json:"initial_balance_fetch_disabled"`
 }
 
 // ReconciliationCoverage is used to add conditions
@@ -303,6 +313,15 @@ type DataConfiguration struct {
 	// wish to use `start_index` at a later point to restart from some
 	// previously synced block.
 	PruningDisabled bool `json:"pruning_disabled"`
+
+	// InitialBalanceFetchDisabled configures rosetta-cli
+	// not to lookup the balance of newly seen accounts at the
+	// parent block before applying operations. Disabling
+	// this step can significantly speed up performance
+	// without impacting validation accuracy (if all genesis
+	// accounts are provided using bootstrap_balances and
+	// syncing starts from genesis).
+	InitialBalanceFetchDisabled bool `json:"initial_balance_fetch_disabled"`
 }
 
 // Configuration contains all configuration settings for running
diff --git a/examples/configuration/default.json b/examples/configuration/default.json
index 8e6692c..4fe1a20 100644
--- a/examples/configuration/default.json
+++ b/examples/configuration/default.json
@@ -34,6 +34,7 @@
   "coin_tracking_disabled": false,
   "status_port": 9090,
   "results_output_file": "",
-  "pruning_disabled": false
+  "pruning_disabled": false,
+  "initial_balance_fetch_disabled": false
  }
 }
\ No newline at end of file
diff --git a/pkg/processor/balance_storage_helper.go b/pkg/processor/balance_storage_helper.go
index 10f2d7e..99b737f 100644
--- a/pkg/processor/balance_storage_helper.go
+++ b/pkg/processor/balance_storage_helper.go
@@ -39,6 +39,7 @@ type BalanceStorageHelper struct {
 	lookupBalanceByBlock bool
 	exemptAccounts       map[string]struct{}
 	balanceExemptions    []*types.BalanceExemption
+	initialFetchDisabled bool
 
 	// Interesting-only Parsing
 	interestingOnly      bool
@@ -53,6 +54,7 @@ func NewBalanceStorageHelper(
 	exemptAccounts []*reconciler.AccountCurrency,
 	interestingOnly bool,
 	balanceExemptions []*types.BalanceExemption,
+	initialFetchDisabled bool,
 ) *BalanceStorageHelper {
 	exemptMap := map[string]struct{}{}
 
@@ -70,6 +72,7 @@ func NewBalanceStorageHelper(
 		interestingAddresses: map[string]struct{}{},
 		interestingOnly:      interestingOnly,
 		balanceExemptions:    balanceExemptions,
+		initialFetchDisabled: initialFetchDisabled,
 	}
 }
 
@@ -83,7 +86,7 @@ func (h *BalanceStorageHelper) AccountBalance(
 	currency *types.Currency,
 	block *types.BlockIdentifier,
 ) (*types.Amount, error) {
-	if !h.lookupBalanceByBlock {
+	if !h.lookupBalanceByBlock || h.initialFetchDisabled {
 		return &types.Amount{
 			Value:    "0",
 			Currency: currency,


Test patch:
diff --git a/pkg/processor/balance_storage_helper_test.go b/pkg/processor/balance_storage_helper_test.go
index a2cd7b5..55aaaba 100644
--- a/pkg/processor/balance_storage_helper_test.go
+++ b/pkg/processor/balance_storage_helper_test.go
@@ -85,6 +85,7 @@ func TestExemptFuncExemptAccounts(t *testing.T) {
 				test.exemptAccounts,
 				false,
 				nil,
+				false,
 			)
 
 			result := helper.ExemptFunc()(&types.Operation{
@@ -127,6 +128,7 @@ func TestExemptFuncInterestingParsing(t *testing.T) {
 				nil,
 				true,
 				nil,
+				false,
 			)
 
 			for _, addr := range test.interestingAddresses {
diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go
index b2c46b1..8745fbe 100644
--- a/pkg/tester/construction.go
+++ b/pkg/tester/construction.go
@@ -102,6 +102,10 @@ func InitializeConstruction(
 		log.Fatalf("%s: unable to get network options", fetchErr.Err.Error())
 	}
 
+	if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Construction.InitialBalanceFetchDisabled {
+		log.Fatal("found balance exemptions but initial balance fetch disabled")
+	}
+
 	counterStorage := storage.NewCounterStorage(localStore)
 	logger := logger.NewLogger(
 		dataPath,
@@ -124,6 +128,7 @@ func InitializeConstruction(
 		nil,
 		true,
 		networkOptions.Allow.BalanceExemptions,
+		config.Construction.InitialBalanceFetchDisabled,
 	)
 
 	balanceStorageHandler := processor.NewBalanceStorageHandler(
diff --git a/pkg/tester/data.go b/pkg/tester/data.go
index efe6ce9..c4e5221 100644
--- a/pkg/tester/data.go
+++ b/pkg/tester/data.go
@@ -230,6 +230,10 @@ func InitializeData(
 		log.Fatalf("%s: unable to get network options", fetchErr.Err.Error())
 	}
 
+	if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Data.InitialBalanceFetchDisabled {
+		log.Fatal("found balance exemptions but initial balance fetch disabled")
+	}
+
 	parser := parser.New(
 		fetcher.Asserter,
 		nil,
@@ -266,6 +270,7 @@ func InitializeData(
 			exemptAccounts,
 			false,
 			networkOptions.Allow.BalanceExemptions,
+			config.Data.InitialBalanceFetchDisabled,
 		)
 
 		balanceStorageHandler := processor.NewBalanceStorageHandler(
@@ -966,6 +971,7 @@ func (t *DataTester) recursiveOpSearch(
 		nil,
 		false,
 		t.parser.BalanceExemptions,
+		false, // we will need to perform an initial balance fetch when finding issues
 	)
 
 	balanceStorageHandler := processor.NewBalanceStorageHandler(