Repository: coinbase/rosetta-cli
Pull request #206 (2020-12-11 21:28:48)

rosetta-sdk-go@v0.6.6
Closes: #205 

This PR adds support for preserving data across workflow executions in the Rosetta DSL and for forcing the retrial of non-200 responses (using `force_retry`).

### Changes
- [x] Update `rosetta-sdk-go@v0.6.6`
- [x] Update `rosetta-cli` version
- [x] Update `CoordinatorHelper`
- [x] Add configuration option for `ForceRetry`
  - [x] Add to `Config.Data`
  - [x] Add to `Config.Construction`
  - [x] Change fetcher initialization to optionally include `WithForceRetry`

### `rosetta-sdk-go` Changes
- [constructor] Add `set_blob` and `get_blob` to DSL [`#277`](https://github.com/coinbase/rosetta-sdk-go/pull/277)
- Bump github.com/tidwall/gjson from 1.6.3 to 1.6.4 [`#270`](https://github.com/coinbase/rosetta-sdk-go/pull/270)
- [constructor] Transactional KV Store [`#276`](https://github.com/coinbase/rosetta-sdk-go/pull/276)
- [fetcher] Optionally Force Retry [`#275`](https://github.com/coinbase/rosetta-sdk-go/pull/275)


Patch:
diff --git a/cmd/check_construction.go b/cmd/check_construction.go
index c26d13b..dc32d3a 100644
--- a/cmd/check_construction.go
+++ b/cmd/check_construction.go
@@ -66,12 +66,19 @@ func runCheckConstructionCmd(cmd *cobra.Command, args []string) error {
 	ensureDataDirectoryExists()
 	ctx, cancel := context.WithCancel(Context)
 
-	fetcher := fetcher.New(
-		Config.OnlineURL,
+	fetcherOpts := []fetcher.Option{
 		fetcher.WithMaxConnections(Config.MaxOnlineConnections),
-		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second),
-		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second),
+		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second),
+		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second),
 		fetcher.WithMaxRetries(Config.MaxRetries),
+	}
+	if Config.ForceRetry {
+		fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry())
+	}
+
+	fetcher := fetcher.New(
+		Config.OnlineURL,
+		fetcherOpts...,
 	)
 
 	_, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network)
diff --git a/cmd/check_data.go b/cmd/check_data.go
index 2277ffd..75b56d0 100644
--- a/cmd/check_data.go
+++ b/cmd/check_data.go
@@ -74,12 +74,19 @@ func runCheckDataCmd(cmd *cobra.Command, args []string) error {
 	ensureDataDirectoryExists()
 	ctx, cancel := context.WithCancel(Context)
 
-	fetcher := fetcher.New(
-		Config.OnlineURL,
+	fetcherOpts := []fetcher.Option{
 		fetcher.WithMaxConnections(Config.MaxOnlineConnections),
-		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second),
-		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second),
+		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second),
+		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second),
 		fetcher.WithMaxRetries(Config.MaxRetries),
+	}
+	if Config.ForceRetry {
+		fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry())
+	}
+
+	fetcher := fetcher.New(
+		Config.OnlineURL,
+		fetcherOpts...,
 	)
 
 	_, _, fetchErr := fetcher.InitializeAsserter(ctx, Config.Network)
diff --git a/cmd/root.go b/cmd/root.go
index dfe2249..866d1ac 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -269,6 +269,6 @@ var versionCmd = &cobra.Command{
 	Use:   "version",
 	Short: "Print rosetta-cli version",
 	Run: func(cmd *cobra.Command, args []string) {
-		fmt.Println("v0.6.4")
+		fmt.Println("v0.6.5")
 	},
 }
diff --git a/cmd/view_balance.go b/cmd/view_balance.go
index 61ca8cb..32ca3da 100644
--- a/cmd/view_balance.go
+++ b/cmd/view_balance.go
@@ -56,11 +56,19 @@ func runViewBalanceCmd(cmd *cobra.Command, args []string) error {
 	}
 
 	// Create a new fetcher
+	fetcherOpts := []fetcher.Option{
+		fetcher.WithMaxConnections(Config.MaxOnlineConnections),
+		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second),
+		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second),
+		fetcher.WithMaxRetries(Config.MaxRetries),
+	}
+	if Config.ForceRetry {
+		fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry())
+	}
+
 	newFetcher := fetcher.New(
 		Config.OnlineURL,
-		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second),
-		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second),
-		fetcher.WithMaxRetries(Config.MaxRetries),
+		fetcherOpts...,
 	)
 
 	// Initialize the fetcher's asserter
diff --git a/cmd/view_block.go b/cmd/view_block.go
index 51262c9..17be0f2 100644
--- a/cmd/view_block.go
+++ b/cmd/view_block.go
@@ -73,11 +73,19 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) error {
 	}
 
 	// Create a new fetcher
+	fetcherOpts := []fetcher.Option{
+		fetcher.WithMaxConnections(Config.MaxOnlineConnections),
+		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second),
+		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second),
+		fetcher.WithMaxRetries(Config.MaxRetries),
+	}
+	if Config.ForceRetry {
+		fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry())
+	}
+
 	newFetcher := fetcher.New(
 		Config.OnlineURL,
-		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second),
-		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second),
-		fetcher.WithMaxRetries(Config.MaxRetries),
+		fetcherOpts...,
 	)
 
 	// Initialize the fetcher's asserter
diff --git a/cmd/view_networks.go b/cmd/view_networks.go
index 6dab0d8..eda06a5 100644
--- a/cmd/view_networks.go
+++ b/cmd/view_networks.go
@@ -41,11 +41,19 @@ not formatted correctly.`,
 )
 
 func runViewNetworksCmd(cmd *cobra.Command, args []string) error {
+	fetcherOpts := []fetcher.Option{
+		fetcher.WithMaxConnections(Config.MaxOnlineConnections),
+		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime) * time.Second),
+		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout) * time.Second),
+		fetcher.WithMaxRetries(Config.MaxRetries),
+	}
+	if Config.ForceRetry {
+		fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry())
+	}
+
 	f := fetcher.New(
 		Config.OnlineURL,
-		fetcher.WithRetryElapsedTime(time.Duration(Config.RetryElapsedTime)*time.Second),
-		fetcher.WithTimeout(time.Duration(Config.HTTPTimeout)*time.Second),
-		fetcher.WithMaxRetries(Config.MaxRetries),
+		fetcherOpts...,
 	)
 
 	// Attempt to fetch network list
diff --git a/configuration/types.go b/configuration/types.go
index 10c73a9..da4d1b2 100644
--- a/configuration/types.go
+++ b/configuration/types.go
@@ -84,6 +84,10 @@ type ConstructionConfiguration struct {
 	// fetcher will open.
 	MaxOfflineConnections int `json:"max_offline_connections"`
 
+	// ForceRetry overrides the default retry handling to retry
+	// on all non-200 responses.
+	ForceRetry bool `json:"force_retry,omitempty"`
+
 	// StaleDepth is the number of blocks to wait before attempting
 	// to rebroadcast after not finding a transaction on-chain.
 	StaleDepth int64 `json:"stale_depth"`
@@ -363,6 +367,10 @@ type Configuration struct {
 	// fetcher will open.
 	MaxOnlineConnections int `json:"max_online_connections"`
 
+	// ForceRetry overrides the default retry handling to retry
+	// on all non-200 responses.
+	ForceRetry bool `json:"force_retry,omitempty"`
+
 	// MaxSyncConcurrency is the maximum sync concurrency to use while syncing blocks.
 	// Sync concurrency is managed automatically by the `syncer` package.
 	MaxSyncConcurrency int64 `json:"max_sync_concurrency"`
diff --git a/go.mod b/go.mod
index f314b4c..73934f7 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli
 go 1.13
 
 require (
-	github.com/coinbase/rosetta-sdk-go v0.6.5
+	github.com/coinbase/rosetta-sdk-go v0.6.6
 	github.com/fatih/color v1.10.0
 	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c
 	github.com/spf13/cobra v1.1.1
diff --git a/go.sum b/go.sum
index c21bece..37c4736 100644
--- a/go.sum
+++ b/go.sum
@@ -80,8 +80,8 @@ github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+
 github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U=
-github.com/coinbase/rosetta-sdk-go v0.6.5 h1:RytFDCPXS64vEYwIOsxsoQGlZZyP9RQvzyYikxymI4w=
-github.com/coinbase/rosetta-sdk-go v0.6.5/go.mod h1:MvQfsL2KlJ5786OdDviRIJE3agui2YcvS1CaQPDl1Yo=
+github.com/coinbase/rosetta-sdk-go v0.6.6 h1:DUuC1LyK1tm/9XzPIuWiC8qOqjaFHvVWH0oA4X4kXJc=
+github.com/coinbase/rosetta-sdk-go v0.6.6/go.mod h1:Spu6Ha6TB3T7TVrzgWzzNvebPuueFbKHyKAfuWQHOE0=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
@@ -367,8 +367,8 @@ github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69
 github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM=
 github.com/tidwall/gjson v1.6.1 h1:LRbvNuNuvAiISWg6gxLEFuCe72UKy5hDqhxW/8183ws=
 github.com/tidwall/gjson v1.6.1/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
-github.com/tidwall/gjson v1.6.3 h1:aHoiiem0dr7GHkW001T1SMTJ7X5PvyekH5WX0whWGnI=
-github.com/tidwall/gjson v1.6.3/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
+github.com/tidwall/gjson v1.6.4 h1:JKsCsJqRVFz8eYCsQ5E/ANRbK6CanAtA9IUvGsXklyo=
+github.com/tidwall/gjson v1.6.4/go.mod h1:BaHyNc5bjzYkPqgLq7mdVzeiRtULKULXLgZFKsxEHI0=
 github.com/tidwall/match v1.0.1 h1:PnKP62LPNxHKTwvHHZZzdOAOCtsJTjo6dZLCwpKm5xc=
 github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E=
 github.com/tidwall/pretty v1.0.2 h1:Z7S3cePv9Jwm1KwS0513MRaoUe3S01WPbLNV40pwWZU=
diff --git a/pkg/processor/coordinator_helper.go b/pkg/processor/coordinator_helper.go
index c017e8c..6d204fc 100644
--- a/pkg/processor/coordinator_helper.go
+++ b/pkg/processor/coordinator_helper.go
@@ -53,6 +53,8 @@ const (
 	argUnsignedTransaction   = "unsigned_transaction"
 	argTransactionIdentifier = "transaction_identifier"
 	argNetworkTransaction    = "network_transaction"
+
+	kvPrefix = "coordinator_kv"
 )
 
 var _ coordinator.Helper = (*CoordinatorHelper)(nil)
@@ -512,3 +514,30 @@ func (c *CoordinatorHelper) HeadBlockExists(ctx context.Context) bool {
 
 	return headBlock != nil
 }
+
+func kvKey(key string) []byte {
+	return []byte(fmt.Sprintf("%s/%s", kvPrefix, key))
+}
+
+// SetBlob transactionally persists
+// a key and value.
+func (c *CoordinatorHelper) SetBlob(
+	ctx context.Context,
+	dbTx database.Transaction,
+	key string,
+	value []byte,
+) error {
+	// We defensively don't claim the value slice
+	// in our buffer pool.
+	return dbTx.Set(ctx, kvKey(key), value, false)
+}
+
+// GetBlob transactionally retrieves
+// a key and value.
+func (c *CoordinatorHelper) GetBlob(
+	ctx context.Context,
+	dbTx database.Transaction,
+	key string,
+) (bool, []byte, error) {
+	return dbTx.Get(ctx, kvKey(key))
+}


Test patch:
diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go
index b6867fd..509a9f1 100644
--- a/pkg/tester/construction.go
+++ b/pkg/tester/construction.go
@@ -162,12 +162,20 @@ func InitializeConstruction(
 		blockStorage,
 		onlineFetcher,
 	)
-	offlineFetcher := fetcher.New(
-		config.Construction.OfflineURL,
+
+	fetcherOpts := []fetcher.Option{
 		fetcher.WithMaxConnections(config.Construction.MaxOfflineConnections),
 		fetcher.WithAsserter(onlineFetcher.Asserter),
-		fetcher.WithTimeout(time.Duration(config.HTTPTimeout)*time.Second),
+		fetcher.WithTimeout(time.Duration(config.HTTPTimeout) * time.Second),
 		fetcher.WithMaxRetries(config.MaxRetries),
+	}
+	if config.Construction.ForceRetry {
+		fetcherOpts = append(fetcherOpts, fetcher.WithForceRetry())
+	}
+
+	offlineFetcher := fetcher.New(
+		config.Construction.OfflineURL,
+		fetcherOpts...,
 	)
 
 	// Import prefunded account and save to database