Repository: coinbase/rosetta-cli
Pull request #184 (2020-11-05 03:20:42)

[reconciler] Inactive Pruning Race
This PR updates the `rosetta-cli` to fix a possible race condition in the `reconciler` related to inactive reconciliations and balance pruning. Details can be found in https://github.com/coinbase/rosetta-sdk-go/pull/221 and https://github.com/coinbase/rosetta-sdk-go/pull/223.

### Changelog
- [x] update rosetta-sdk-go
- [x] add `MaxReorgDepth` to config file


Patch:
diff --git a/configuration/configuration.go b/configuration/configuration.go
index c227dea..1f6df0d 100644
--- a/configuration/configuration.go
+++ b/configuration/configuration.go
@@ -53,6 +53,7 @@ func DefaultConfiguration() *Configuration {
 		MaxRetries:           DefaultMaxRetries,
 		MaxSyncConcurrency:   DefaultMaxSyncConcurrency,
 		TipDelay:             DefaultTipDelay,
+		MaxReorgDepth:        DefaultMaxReorgDepth,
 		Data:                 DefaultDataConfiguration(),
 	}
 }
@@ -148,6 +149,10 @@ func populateMissingFields(config *Configuration) *Configuration {
 		config.TipDelay = DefaultTipDelay
 	}
 
+	if config.MaxReorgDepth == 0 {
+		config.MaxReorgDepth = DefaultMaxReorgDepth
+	}
+
 	config.Construction = populateConstructionMissingFields(config.Construction)
 	config.Data = populateDataMissingFields(config.Data)
 
diff --git a/configuration/types.go b/configuration/types.go
index 9a6a51d..8feb383 100644
--- a/configuration/types.go
+++ b/configuration/types.go
@@ -59,6 +59,7 @@ const (
 	DefaultTipDelay                          = 300
 	DefaultBlockBroadcastLimit               = 5
 	DefaultStatusPort                        = 9090
+	DefaultMaxReorgDepth                     = 100
 
 	// ETH Defaults
 	EthereumIDBlockchain = "Ethereum"
@@ -366,6 +367,15 @@ type Configuration struct {
 	// we are considered to be behind tip.
 	TipDelay int64 `json:"tip_delay"`
 
+	// MaxReorgDepth specifies the maximum possible reorg depth of the blockchain
+	// being synced. This value is used to determine how aggressively to prune
+	// old block data.
+	//
+	// It is better to be overly cautious here as keeping a few
+	// too many blocks around is much better than running into an
+	// error caused by missing block data!
+	MaxReorgDepth int `json:"max_reorg_depth,omitempty"`
+
 	// LogConfiguration determines if the configuration settings
 	// should be printed to the console when a file is loaded.
 	LogConfiguration bool `json:"log_configuration"`
diff --git a/examples/configuration/default.json b/examples/configuration/default.json
index 4fe1a20..1d5a448 100644
--- a/examples/configuration/default.json
+++ b/examples/configuration/default.json
@@ -11,6 +11,7 @@
  "max_online_connections": 120,
  "max_sync_concurrency": 64,
  "tip_delay": 300,
+ "max_reorg_depth": 100,
  "log_configuration": false,
  "compression_disabled": false,
  "memory_limit_disabled": false,
diff --git a/go.mod b/go.mod
index 5178e24..a0f910e 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli
 go 1.13
 
 require (
-	github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79
+	github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105044327-5d14e78bb3c7
 	github.com/fatih/color v1.10.0
 	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c
 	github.com/spf13/cobra v1.1.1
diff --git a/go.sum b/go.sum
index 2d27626..b740f0f 100644
--- a/go.sum
+++ b/go.sum
@@ -80,6 +80,12 @@ github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202 h1:tSzJ9
 github.com/coinbase/rosetta-sdk-go v0.5.9-0.20201030213420-e8c5df9fe202/go.mod h1:xd4wYUhV3LkY78SPH8BUhc88rXfn2jYgN9BfiSjbcvM=
 github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79 h1:UvuAOEDni8pSopTNfrZHel9EEPzvPxLD8bjqMmP23cc=
 github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201103220049-3a720608da79/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4=
+github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f h1:/w0ECMQ4kpd0vPitSOzkKteGSxmbxPACcJEbQHMfEf8=
+github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105031224-eeb3d969685f/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4=
+github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e h1:g66k0AI2GvNuop+khjY0qO72l7JIO6ma1QbH6fJLnn8=
+github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105041044-671e2d01699e/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4=
+github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105044327-5d14e78bb3c7 h1:Y6Kk2bar8EVLyMLNOQ76h7OSeP4+XJx6GHhn11YDwew=
+github.com/coinbase/rosetta-sdk-go v0.5.10-0.20201105044327-5d14e78bb3c7/go.mod h1:pqBibzTcpz0mLqb7is8fo2qf93tcHjItJ3u7Yud8RI4=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=


Test patch:
diff --git a/configuration/configuration_test.go b/configuration/configuration_test.go
index 84610f5..7cc2c50 100644
--- a/configuration/configuration_test.go
+++ b/configuration/configuration_test.go
@@ -67,6 +67,7 @@ var (
 		MaxRetries:           1000,
 		MaxSyncConcurrency:   12,
 		TipDelay:             1231,
+		MaxReorgDepth:        12,
 		Construction: &ConstructionConfiguration{
 			OfflineURL:            "https://ashdjaksdkjshdk",
 			MaxOfflineConnections: 21,
diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go
index 625375a..194198a 100644
--- a/pkg/tester/construction.go
+++ b/pkg/tester/construction.go
@@ -258,6 +258,7 @@ func InitializeConstruction(
 		[]storage.BlockWorker{balanceStorage, coinStorage, broadcastStorage},
 		syncer.DefaultCacheSize,
 		config.MaxSyncConcurrency,
+		config.MaxReorgDepth,
 	)
 
 	return &ConstructionTester{
diff --git a/pkg/tester/data.go b/pkg/tester/data.go
index f8df23d..068b30b 100644
--- a/pkg/tester/data.go
+++ b/pkg/tester/data.go
@@ -317,6 +317,7 @@ func InitializeData(
 		blockWorkers,
 		syncer.DefaultCacheSize,
 		config.MaxSyncConcurrency,
+		config.MaxReorgDepth,
 	)
 
 	return &DataTester{
@@ -381,7 +382,7 @@ func (t *DataTester) PruneableIndex(
 	// balances at their index.
 	//
 	// It is ok if the returned value here is negative.
-	return headIndex - statefulsyncer.DefaultPruningDepth, nil
+	return headIndex - int64(t.config.MaxReorgDepth), nil
 }
 
 // StartReconciler starts the reconciler if
@@ -1017,6 +1018,7 @@ func (t *DataTester) recursiveOpSearch(
 		[]storage.BlockWorker{balanceStorage},
 		syncer.DefaultCacheSize,
 		t.config.MaxSyncConcurrency,
+		t.config.MaxReorgDepth,
 	)
 
 	g, ctx := errgroup.WithContext(ctx)