Repository: coinbase/rosetta-cli
Pull request #178 (2020-10-30 21:40:07)

[reconciler] Reduced Error Sensitivity at Tip
This PR updates the `rosetta-cli` to "skip" balance lookups that fail at tip. This can be caused by any number of race conditions related to reorgs and it is better to skip them than to return a false positive error.

### Changes
- [x] Update rosetta-sdk-go (https://github.com/coinbase/rosetta-sdk-go/pull/215, https://github.com/coinbase/rosetta-sdk-go/pull/216)
- [x] Update rosetta-cli version
- [x] Log blocks to console when `log_blocks` is enabled



Patch:
diff --git a/cmd/root.go b/cmd/root.go
index d89c144..4ea78f2 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -257,6 +257,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print rosetta-cli version",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("v0.5.19")
+		fmt.Println("v0.5.20")
 	},
 }
diff --git a/configuration/configuration.go b/configuration/configuration.go
index 0908f73..c227dea 100644
--- a/configuration/configuration.go
+++ b/configuration/configuration.go
@@ -250,7 +250,10 @@ func assertDataConfiguration(config *DataConfiguration) error {
 
 		accountCount := config.EndConditions.ReconciliationCoverage.AccountCount
 		if accountCount != nil && *accountCount < 0 {
-			return fmt.Errorf("reconciliation coverage account count %d must be >= 0", *accountCount)
+			return fmt.Errorf(
+				"reconciliation coverage account count %d must be >= 0",
+				*accountCount,
+			)
 		}
 
 		if config.BalanceTrackingDisabled {
@@ -308,7 +311,10 @@ func modifyFilePaths(config *Configuration, fileDir string) {
 
 	if config.Construction != nil {
 		if len(config.Construction.ConstructorDSLFile) > 0 {
-			config.Construction.ConstructorDSLFile = path.Join(fileDir, config.Construction.ConstructorDSLFile)
+			config.Construction.ConstructorDSLFile = path.Join(
+				fileDir,
+				config.Construction.ConstructorDSLFile,
+			)
 		}
 	}
 }
diff --git a/go.mod b/go.mod
index ada68e6..bda12ee 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli
 go 1.13
 
 require (
-	github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6
+	github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202
 	github.com/fatih/color v1.9.0
 	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c
 	github.com/spf13/cobra v1.1.1
diff --git a/go.sum b/go.sum
index 5e99c3e..57a001f 100644
--- a/go.sum
+++ b/go.sum
@@ -76,8 +76,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U=
-github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6 h1:5zDhG7QoXRf2GtNeDqfnUdAe5gm1fdKy0h5Txa7NHec=
-github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201029210921-d7499a34c1f6/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM=
+github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 h1:tSzJ9tXxfa9UsXygm0HQL3SMhL2MTamT9nPKVb+6LG4=
+github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -511,6 +511,7 @@ golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69 h1:yBHHx+XZqXJBm6Exke3N7V9gnlsyXxoCPEb1yVenjfk=
 golang.org/x/tools v0.0.0-20200117012304-6edc0a871e69/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
diff --git a/pkg/logger/logger.go b/pkg/logger/logger.go
index f614271..07f8f11 100644
--- a/pkg/logger/logger.go
+++ b/pkg/logger/logger.go
@@ -143,7 +143,10 @@ func (l *Logger) LogDataStatus(ctx context.Context, status *results.CheckDataSta
 }
 
 // LogConstructionStatus logs results.CheckConstructionStatus.
-func (l *Logger) LogConstructionStatus(ctx context.Context, status *results.CheckConstructionStatus) {
+func (l *Logger) LogConstructionStatus(
+	ctx context.Context,
+	status *results.CheckConstructionStatus,
+) {
 	statsMessage := fmt.Sprintf(
 		"[STATS] Transactions Confirmed: %d (Created: %d, In Progress: %d, Stale: %d, Failed: %d) Addresses Created: %d",
 		status.Stats.TransactionsConfirmed,
@@ -196,15 +199,16 @@ func (l *Logger) AddBlockStream(
 
 	defer closeFile(f)
 
-	_, err = f.WriteString(fmt.Sprintf(
+	blockString := fmt.Sprintf(
 		"%s Block %d:%s with Parent Block %d:%s\n",
 		addEvent,
 		block.BlockIdentifier.Index,
 		block.BlockIdentifier.Hash,
 		block.ParentBlockIdentifier.Index,
 		block.ParentBlockIdentifier.Hash,
-	))
-	if err != nil {
+	)
+	fmt.Print(blockString)
+	if _, err := f.WriteString(blockString); err != nil {
 		return err
 	}
 
@@ -232,17 +236,15 @@ func (l *Logger) RemoveBlockStream(
 
 	defer closeFile(f)
 
-	_, err = f.WriteString(fmt.Sprintf(
+	blockString := fmt.Sprintf(
 		"%s Block %d:%s\n",
 		removeEvent,
 		block.Index,
 		block.Hash,
-	))
-	if err != nil {
-		return err
-	}
-
-	return nil
+	)
+	fmt.Print(blockString)
+	_, err = f.WriteString(blockString)
+	return err
 }
 
 // TransactionStream writes the next processed block's transactions
diff --git a/pkg/processor/reconciler_helper.go b/pkg/processor/reconciler_helper.go
index ddcee94..e114245 100644
--- a/pkg/processor/reconciler_helper.go
+++ b/pkg/processor/reconciler_helper.go
@@ -35,6 +35,7 @@ type ReconcilerHelper struct {
 	database       storage.Database
 	blockStorage   *storage.BlockStorage
 	balanceStorage *storage.BalanceStorage
+	tipDelay       int64
 }
 
 // NewReconcilerHelper returns a new ReconcilerHelper.
@@ -44,6 +45,7 @@ func NewReconcilerHelper(
 	database storage.Database,
 	blockStorage *storage.BlockStorage,
 	balanceStorage *storage.BalanceStorage,
+	tipDelay int64,
 ) *ReconcilerHelper {
 	return &ReconcilerHelper{
 		network:        network,
@@ -51,6 +53,7 @@ func NewReconcilerHelper(
 		database:       database,
 		blockStorage:   blockStorage,
 		balanceStorage: balanceStorage,
+		tipDelay:       tipDelay,
 	}
 }
 
@@ -73,6 +76,18 @@ func (h *ReconcilerHelper) CanonicalBlock(
 	return h.blockStorage.CanonicalBlockTransactional(ctx, block, dbTx)
 }
 
+// IndexAtTip returns a boolean indicating if a block
+// index is at tip (provided some acceptable
+// tip delay). If the index is ahead of the head block
+// and the head block is at tip, we consider the
+// index at tip.
+func (h *ReconcilerHelper) IndexAtTip(
+	ctx context.Context,
+	index int64,
+) (bool, error) {
+	return h.blockStorage.IndexAtTip(ctx, h.tipDelay, index)
+}
+
 // CurrentBlock returns the last processed block and is used
 // to determine which block to check account balances at during
 // inactive reconciliation.
diff --git a/pkg/results/data_results.go b/pkg/results/data_results.go
index bf7f5c7..950c97c 100644
--- a/pkg/results/data_results.go
+++ b/pkg/results/data_results.go
@@ -334,17 +334,31 @@ func ComputeCheckDataProgress(
 		return nil
 	}
 
-	blocksPerSecond := new(big.Float).Quo(new(big.Float).SetInt64(adjustedBlocks), new(big.Float).SetInt(elapsedTime))
+	blocksPerSecond := new(
+		big.Float,
+	).Quo(
+		new(big.Float).SetInt64(adjustedBlocks),
+		new(big.Float).SetInt(elapsedTime),
+	)
 	blocksPerSecondFloat, _ := blocksPerSecond.Float64()
-	blocksSynced := new(big.Float).Quo(new(big.Float).SetInt64(headBlock.Index), new(big.Float).SetInt64(tipIndex))
+	blocksSynced := new(
+		big.Float,
+	).Quo(
+		new(big.Float).SetInt64(headBlock.Index),
+		new(big.Float).SetInt64(tipIndex),
+	)
 	blocksSyncedFloat, _ := blocksSynced.Float64()
 
 	return &CheckDataProgress{
-		Blocks:              headBlock.Index,
-		Tip:                 tipIndex,
-		Completed:           blocksSyncedFloat * utils.OneHundred,
-		Rate:                blocksPerSecondFloat,
-		TimeRemaining:       utils.TimeToTip(blocksPerSecondFloat, headBlock.Index, tipIndex).String(),
+		Blocks:    headBlock.Index,
+		Tip:       tipIndex,
+		Completed: blocksSyncedFloat * utils.OneHundred,
+		Rate:      blocksPerSecondFloat,
+		TimeRemaining: utils.TimeToTip(
+			blocksPerSecondFloat,
+			headBlock.Index,
+			tipIndex,
+		).String(),
 		ReconcilerQueueSize: reconciler.QueueSize(),
 		ReconcilerLastIndex: reconciler.LastIndexReconciled(),
 	}
@@ -604,7 +618,12 @@ func ComputeCheckDataTests( // nolint:gocognit
 		ResponseAssertion: ResponseAssertionTest(err),
 		BlockSyncing:      BlockSyncingTest(err, blocksSynced),
 		BalanceTracking:   BalanceTrackingTest(cfg, err, operationsSeen),
-		Reconciliation:    ReconciliationTest(cfg, err, reconciliationsPerformed, reconciliationsFailed),
+		Reconciliation: ReconciliationTest(
+			cfg,
+			err,
+			reconciliationsPerformed,
+			reconciliationsFailed,
+		),
 	}
 }
 


Test patch:
diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go
index 8745fbe..625375a 100644
--- a/pkg/tester/construction.go
+++ b/pkg/tester/construction.go
@@ -102,7 +102,8 @@ func InitializeConstruction(
 		log.Fatalf("%s: unable to get network options", fetchErr.Err.Error())
 	}
 
-	if len(networkOptions.Allow.BalanceExemptions) > 0 && config.Construction.InitialBalanceFetchDisabled {
+	if len(networkOptions.Allow.BalanceExemptions) > 0 &&
+		config.Construction.InitialBalanceFetchDisabled {
 		log.Fatal("found balance exemptions but initial balance fetch disabled")
 	}
 
@@ -296,7 +297,13 @@ func (t *ConstructionTester) StartPeriodicLogger(
 		case <-ctx.Done():
 			return ctx.Err()
 		case <-tc.C:
-			status := results.ComputeCheckConstructionStatus(ctx, t.config, t.counterStorage, t.broadcastStorage, t.jobStorage)
+			status := results.ComputeCheckConstructionStatus(
+				ctx,
+				t.config,
+				t.counterStorage,
+				t.broadcastStorage,
+				t.jobStorage,
+			)
 			t.logger.LogConstructionStatus(ctx, status)
 		}
 	}
diff --git a/pkg/tester/data.go b/pkg/tester/data.go
index be3983d..8a38274 100644
--- a/pkg/tester/data.go
+++ b/pkg/tester/data.go
@@ -211,6 +211,7 @@ func InitializeData(
 		localStore,
 		blockStorage,
 		balanceStorage,
+		config.TipDelay,
 	)
 
 	reconcilerHandler := processor.NewReconcilerHandler(
@@ -732,7 +733,10 @@ func (t *DataTester) WaitForEmptyQueue(
 
 // DrainReconcilerQueue returns once the reconciler queue has been drained
 // or an error is encountered.
-func (t *DataTester) DrainReconcilerQueue(ctx context.Context, sigListeners *[]context.CancelFunc) error {
+func (t *DataTester) DrainReconcilerQueue(
+	ctx context.Context,
+	sigListeners *[]context.CancelFunc,
+) error {
 	color.Cyan("draining reconciler backlog (you can disable this in your configuration file)")
 
 	// To cancel all execution, need to call multiple cancel functions.
@@ -801,7 +805,9 @@ func (t *DataTester) HandleErr(err error, sigListeners *[]context.CancelFunc) er
 		if shouldReconcile(t.config) &&
 			t.reconciler.QueueSize() > 0 {
 			if t.config.Data.ReconciliationDrainDisabled {
-				color.Cyan("skipping reconciler backlog drain (you can enable this in your configuration file)")
+				color.Cyan(
+					"skipping reconciler backlog drain (you can enable this in your configuration file)",
+				)
 			} else {
 				drainErr := t.DrainReconcilerQueue(ctx, sigListeners)
 				if drainErr != nil {
@@ -954,6 +960,7 @@ func (t *DataTester) recursiveOpSearch(
 		localStore,
 		blockStorage,
 		balanceStorage,
+		t.config.TipDelay,
 	)
 
 	reconcilerHandler := processor.NewReconcilerHandler(