Repository: coinbase/rosetta-cli
Pull request #131 (2020-09-12 01:15:08)

Infer Historical Balance Lookup
Closes: #43 

This PR adds the ability for `check:data` to infer whether historical balance lookup is enabled from the `/network/options` response of a Rosetta API implementation.


Patch:
diff --git a/.circleci/config.yml b/.circleci/config.yml
index 79097af..d0c4311 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -50,7 +50,7 @@ jobs:
       name: default
     steps:
       - *fast-checkout
-      - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.24.0
+      - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0
       - run: make lint
   check-license:
     executor:
diff --git a/README.md b/README.md
index a04f5ac..0300c5d 100644
--- a/README.md
+++ b/README.md
@@ -38,6 +38,12 @@ The binary will be installed inside the `./bin` directory (relative to where the
 
 _Downloading binaries from the Github UI will cause permission errors on Mac._
 
+### Installing in Custom Location
+To download the binary into a specific directory, run:
+```
+curl -sSfL https://raw.githubusercontent.com/coinbase/rosetta-cli/master/scripts/install.sh | sh -s -- -b <relative directory>
+```
+
 ## Usage
 ```
 CLI for the Rosetta API
diff --git a/configuration/configuration.go b/configuration/configuration.go
index 9a63676..fde818d 100644
--- a/configuration/configuration.go
+++ b/configuration/configuration.go
@@ -230,11 +230,11 @@ type DataConfiguration struct {
 	// it will be ignored.
 	BootstrapBalances string `json:"bootstrap_balances"`
 
-	// HistoricalBalanceDisabled is a boolean that dictates how balance lookup is performed.
+	// HistoricalBalanceEnabled is a boolean that dictates how balance lookup is performed.
 	// When set to true, balances are looked up at the block where a balance
 	// change occurred instead of at the current block. Blockchains that do not support
 	// historical balance lookup should set this to false.
-	HistoricalBalanceDisabled bool `json:"historical_balance_disabled"`
+	HistoricalBalanceEnabled *bool `json:"historical_balance_enabled,omitempty"`
 
 	// InterestingAccounts is a path to a file listing all accounts to check on each block. Look
 	// at the examples directory for an example of how to structure this file.
diff --git a/examples/configuration/bitcoin.json b/examples/configuration/bitcoin.json
index ff109df..0b6750e 100644
--- a/examples/configuration/bitcoin.json
+++ b/examples/configuration/bitcoin.json
@@ -199,7 +199,6 @@
   "ignore_reconciliation_error": false,
   "exempt_accounts": "",
   "bootstrap_balances": "",
-  "historical_balance_disabled": true,
   "interesting_accounts": "",
   "reconciliation_disabled": false,
   "inactive_discrepency_search_disabled": false,
@@ -210,4 +209,4 @@
   },
   "results_output_file": ""
  }
-}
\ No newline at end of file
+}
diff --git a/examples/configuration/default.json b/examples/configuration/default.json
index 5413458..badbe81 100644
--- a/examples/configuration/default.json
+++ b/examples/configuration/default.json
@@ -24,7 +24,6 @@
   "ignore_reconciliation_error": false,
   "exempt_accounts": "",
   "bootstrap_balances": "",
-  "historical_balance_disabled": false,
   "interesting_accounts": "",
   "reconciliation_disabled": false,
   "inactive_discrepency_search_disabled": false,
diff --git a/examples/configuration/ethereum.json b/examples/configuration/ethereum.json
index cf777e3..0a6ef24 100644
--- a/examples/configuration/ethereum.json
+++ b/examples/configuration/ethereum.json
@@ -166,7 +166,6 @@
   "ignore_reconciliation_error": false,
   "exempt_accounts": "",
   "bootstrap_balances": "",
-  "historical_balance_disabled": false,
   "interesting_accounts": "",
   "reconciliation_disabled": false,
   "inactive_discrepency_search_disabled": false,
@@ -177,4 +176,4 @@
   },
   "results_output_file": ""
  }
-}
\ No newline at end of file
+}
diff --git a/examples/configuration/simple.json b/examples/configuration/simple.json
index 1237cc7..cbf8755 100644
--- a/examples/configuration/simple.json
+++ b/examples/configuration/simple.json
@@ -8,7 +8,7 @@
  "http_timeout": 10,
  "tip_delay": 300,
  "data": {
-  "historical_balance_disabled": true,
+  "historical_balance_enabled": false,
   "reconciliation_disabled": true,
   "inactive_discrepency_search_disabled": true,
   "balance_tracking_disabled": true,
diff --git a/go.mod b/go.mod
index 14db5cb..e7b82bb 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli
 go 1.13
 
 require (
-	github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4
+	github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014
 	github.com/fatih/color v1.9.0
 	github.com/jinzhu/copier v0.0.0-20190924061706-b57f9002281a
 	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c
diff --git a/go.sum b/go.sum
index 3f7aace..f3d7fee 100644
--- a/go.sum
+++ b/go.sum
@@ -125,6 +125,8 @@ github.com/coinbase/rosetta-sdk-go v0.4.1 h1:bL8qgwRZ6O56V+drjbdRAhnPZYdkNFjmY5V
 github.com/coinbase/rosetta-sdk-go v0.4.1/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
 github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4 h1:aEQB1Y2qp036Cfy+Cm2pjtPEKmFofw/QKzfd3JL/wIU=
 github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200911162523-bbfbd83145d4/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
+github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014 h1:vMWAMtcGnpEhWh6kEDzDqviivHwFRp9w01MXGCLJCMU=
+github.com/coinbase/rosetta-sdk-go v0.4.2-0.20200912010944-f7d13f597014/go.mod h1:Luv0AhzZH81eul2hYZ3w0hBGwmFPiexwbntYxihEZck=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=


Test patch:
diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go
index 0fae220..2994a86 100644
--- a/configuration/configuration_test.go
+++ b/configuration/configuration_test.go
@@ -27,12 +27,13 @@ import (
 )
 
 var (
-	startIndex    = int64(89)
-	badStartIndex = int64(-10)
-	goodCoverage  = float64(0.33)
-	badCoverage   = float64(-2)
-	endTip        = false
-	fakeWorkflows = []*job.Workflow{
+	startIndex        = int64(89)
+	badStartIndex     = int64(-10)
+	goodCoverage      = float64(0.33)
+	badCoverage       = float64(-2)
+	endTip            = false
+	historicalEnabled = true
+	fakeWorkflows     = []*job.Workflow{
 		{
 			Name:        string(job.CreateAccount),
 			Concurrency: job.ReservedWorkflowConcurrency,
@@ -72,7 +73,7 @@ var (
 			InactiveReconciliationConcurrency: 2938,
 			InactiveReconciliationFrequency:   3,
 			ReconciliationDisabled:            false,
-			HistoricalBalanceDisabled:         true,
+			HistoricalBalanceEnabled:          &historicalEnabled,
 			StartIndex:                        &startIndex,
 			EndConditions: &DataEndConditions{
 				ReconciliationCoverage: &goodCoverage,
diff --git a/pkg/tester/data.go b/pkg/tester/data.go
index 25c7ba9..ab618a4 100644
--- a/pkg/tester/data.go
+++ b/pkg/tester/data.go
@@ -64,20 +64,21 @@ const (
 
 // DataTester coordinates the `check:data` test.
 type DataTester struct {
-	network           *types.NetworkIdentifier
-	database          storage.Database
-	config            *configuration.Configuration
-	syncer            *statefulsyncer.StatefulSyncer
-	reconciler        *reconciler.Reconciler
-	logger            *logger.Logger
-	balanceStorage    *storage.BalanceStorage
-	blockStorage      *storage.BlockStorage
-	counterStorage    *storage.CounterStorage
-	reconcilerHandler *processor.ReconcilerHandler
-	fetcher           *fetcher.Fetcher
-	signalReceived    *bool
-	genesisBlock      *types.BlockIdentifier
-	cancel            context.CancelFunc
+	network                  *types.NetworkIdentifier
+	database                 storage.Database
+	config                   *configuration.Configuration
+	syncer                   *statefulsyncer.StatefulSyncer
+	reconciler               *reconciler.Reconciler
+	logger                   *logger.Logger
+	balanceStorage           *storage.BalanceStorage
+	blockStorage             *storage.BlockStorage
+	counterStorage           *storage.CounterStorage
+	reconcilerHandler        *processor.ReconcilerHandler
+	fetcher                  *fetcher.Fetcher
+	signalReceived           *bool
+	genesisBlock             *types.BlockIdentifier
+	cancel                   context.CancelFunc
+	historicalBalanceEnabled bool
 
 	endCondition       configuration.CheckDataEndCondition
 	endConditionDetail string
@@ -197,12 +198,25 @@ func InitializeData(
 		log.Fatalf("%s: unable to get previously seen accounts", err.Error())
 	}
 
+	// Determine if we should perform historical balance lookups
+	var historicalBalanceEnabled bool
+	if config.Data.HistoricalBalanceEnabled != nil {
+		historicalBalanceEnabled = *config.Data.HistoricalBalanceEnabled
+	} else { // we must look it up
+		networkOptions, fetchErr := fetcher.NetworkOptionsRetry(ctx, network, nil)
+		if err != nil {
+			log.Fatalf("%s: unable to get network options", fetchErr.Err.Error())
+		}
+
+		historicalBalanceEnabled = networkOptions.Allow.HistoricalBalanceLookup
+	}
+
 	r := reconciler.New(
 		reconcilerHelper,
 		reconcilerHandler,
 		reconciler.WithActiveConcurrency(int(config.Data.ActiveReconciliationConcurrency)),
 		reconciler.WithInactiveConcurrency(int(config.Data.InactiveReconciliationConcurrency)),
-		reconciler.WithLookupBalanceByBlock(!config.Data.HistoricalBalanceDisabled),
+		reconciler.WithLookupBalanceByBlock(historicalBalanceEnabled),
 		reconciler.WithInterestingAccounts(interestingAccounts),
 		reconciler.WithSeenAccounts(seenAccounts),
 		reconciler.WithDebugLogging(config.Data.LogReconciliations),
@@ -214,7 +228,7 @@ func InitializeData(
 		balanceStorageHelper := processor.NewBalanceStorageHelper(
 			network,
 			fetcher,
-			!config.Data.HistoricalBalanceDisabled,
+			historicalBalanceEnabled,
 			exemptAccounts,
 			false,
 		)
@@ -269,20 +283,21 @@ func InitializeData(
 	)
 
 	return &DataTester{
-		network:           network,
-		database:          localStore,
-		config:            config,
-		syncer:            syncer,
-		cancel:            cancel,
-		reconciler:        r,
-		logger:            logger,
-		balanceStorage:    balanceStorage,
-		blockStorage:      blockStorage,
-		counterStorage:    counterStorage,
-		reconcilerHandler: reconcilerHandler,
-		fetcher:           fetcher,
-		signalReceived:    signalReceived,
-		genesisBlock:      genesisBlock,
+		network:                  network,
+		database:                 localStore,
+		config:                   config,
+		syncer:                   syncer,
+		cancel:                   cancel,
+		reconciler:               r,
+		logger:                   logger,
+		balanceStorage:           balanceStorage,
+		blockStorage:             blockStorage,
+		counterStorage:           counterStorage,
+		reconcilerHandler:        reconcilerHandler,
+		fetcher:                  fetcher,
+		signalReceived:           signalReceived,
+		genesisBlock:             genesisBlock,
+		historicalBalanceEnabled: historicalBalanceEnabled,
 	}
 }
 
@@ -501,9 +516,9 @@ func (t *DataTester) HandleErr(ctx context.Context, err error, sigListeners []co
 		ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "")
 	}
 
-	if t.config.Data.HistoricalBalanceDisabled {
+	if !t.historicalBalanceEnabled {
 		color.Yellow(
-			"Can't find the block missing operations automatically, please enable --lookup-balance-by-block",
+			"Can't find the block missing operations automatically, please enable historical balance lookup",
 		)
 		ExitData(t.config, t.counterStorage, t.balanceStorage, err, 1, "", "")
 	}
@@ -613,14 +628,14 @@ func (t *DataTester) recursiveOpSearch(
 		// Do not do any inactive lookups when looking for the block with missing
 		// operations.
 		reconciler.WithInactiveConcurrency(0),
-		reconciler.WithLookupBalanceByBlock(!t.config.Data.HistoricalBalanceDisabled),
+		reconciler.WithLookupBalanceByBlock(t.historicalBalanceEnabled),
 		reconciler.WithInterestingAccounts([]*reconciler.AccountCurrency{accountCurrency}),
 	)
 
 	balanceStorageHelper := processor.NewBalanceStorageHelper(
 		t.network,
 		t.fetcher,
-		!t.config.Data.HistoricalBalanceDisabled,
+		t.historicalBalanceEnabled,
 		nil,
 		false,
 	)