Repository: coinbase/rosetta-cli
Pull request #310 (2022-05-10 17:58:49)

Added benchmarking for rosetta server endpoints through check:perf co…
<img width="611" alt="Screen Shot 2022-04-27 at 1 56 35 PM" src="https://user-images.githubusercontent.com/29065066/167697578-aa17d7e9-2e25-442e-95d8-24661feeec8c.png">

Benchmarking for /Block and /Account/Balance endpoints

### Motivation
To prevent performance degradations in Rosetta Servers

cb-heimdall: Review Error for shiatcb @ 2022-05-11 17:20:30 UTC
User must have write permissions to review

Patch:
diff --git a/cmd/check_perf.go b/cmd/check_perf.go
new file mode 100644
index 0000000..613503a
--- /dev/null
+++ b/cmd/check_perf.go
@@ -0,0 +1,65 @@
+// Copyright 2022 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/coinbase/rosetta-cli/pkg/results"
+	t "github.com/coinbase/rosetta-cli/pkg/tester"
+	"github.com/spf13/cobra"
+	"golang.org/x/sync/errgroup"
+)
+
+var (
+	checkPerfCmd = &cobra.Command{
+		Use:   "check:perf",
+		Short: "Benchmark performance of time-critical endpoints of Asset Issuer's Rosetta Implementation",
+		Long: `This command can be used to benchmark the performance of time critical methods for a Rosetta server.
+This is useful for ensuring that there are no performance degradations in the rosetta-server.`,
+		RunE: runCheckPerfCmd,
+	}
+)
+
+func runCheckPerfCmd(_ *cobra.Command, _ []string) error {
+	ctx, cancel := context.WithCancel(Context)
+	defer cancel()
+	g, ctx := errgroup.WithContext(ctx)
+
+	TotalNumEndpoints := int64(Config.Perf.NumTimesToHitEndpoints) * (Config.Perf.EndBlock - Config.Perf.StartBlock)
+	perfRawStats := &results.CheckPerfRawStats{AccountBalanceEndpointTotalTime: -1, BlockEndpointTotalTime: -1}
+
+	fmt.Printf("Running Check:Perf for %s:%s for blocks %d-%d \n", Config.Network.Blockchain, Config.Network.Network, Config.Perf.StartBlock, Config.Perf.EndBlock)
+
+	fetcher, timer, elapsed := t.SetupBenchmarking(Config)
+	blockEndpointTimeConstraint := time.Duration(Config.Perf.BlockEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond
+	blockEndpointCtx, blockEndpointCancel := context.WithTimeout(ctx, blockEndpointTimeConstraint)
+	g.Go(func() error {
+		return t.BmarkBlock(blockEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats)
+	})
+	defer blockEndpointCancel()
+
+	fetcher, timer, elapsed = t.SetupBenchmarking(Config)
+	accountBalanceEndpointTimeConstraint := time.Duration(Config.Perf.AccountBalanceEndpointTimeConstraintMs*TotalNumEndpoints) * time.Millisecond
+	accountBalanceEndpointCtx, accountBalanceEndpointCancel := context.WithTimeout(ctx, accountBalanceEndpointTimeConstraint)
+	g.Go(func() error {
+		return t.BmarkAccountBalance(accountBalanceEndpointCtx, Config, fetcher, timer, elapsed, perfRawStats)
+	})
+	defer accountBalanceEndpointCancel()
+
+	return results.ExitPerf(Config.Perf, g.Wait(), perfRawStats)
+}
diff --git a/cmd/root.go b/cmd/root.go
index 649bd2f..1c41844 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -240,6 +240,9 @@ default values.`,
 	// Utils
 	rootCmd.AddCommand(utilsAsserterConfigurationCmd)
 	rootCmd.AddCommand(utilsTrainZstdCmd)
+
+	// Benchmark commands
+	rootCmd.AddCommand(checkPerfCmd)
 }
 
 func initConfig() {
diff --git a/configuration/configuration.go b/configuration/configuration.go
index c92c2cd..3b685a4 100644
--- a/configuration/configuration.go
+++ b/configuration/configuration.go
@@ -44,6 +44,19 @@ func DefaultDataConfiguration() *DataConfiguration {
 	}
 }
 
+// DefaultPerfConfiguration returns the default *CheckPerfConfiguration
+// for running `check:perf`.
+func DefaultPerfConfiguration() *CheckPerfConfiguration {
+	return &CheckPerfConfiguration{
+		StartBlock:                             10,
+		BlockEndpointTimeConstraintMs:          50000000,
+		AccountBalanceEndpointTimeConstraintMs: 50000000,
+		EndBlock:                               50,
+		NumTimesToHitEndpoints:                 1,
+		StatsOutputFile:                        "./check_perf_stats.json",
+	}
+}
+
 // DefaultConfiguration returns a *Configuration with the
 // EthereumNetwork, DefaultURL, DefaultTimeout,
 // DefaultConstructionConfiguration and DefaultDataConfiguration.
@@ -61,6 +74,24 @@ func DefaultConfiguration() *Configuration {
 	}
 }
 
+func populatePerfMissingFields(
+	perfConfig *CheckPerfConfiguration,
+) *CheckPerfConfiguration {
+	if perfConfig == nil {
+		return nil
+	}
+
+	if len(perfConfig.StatsOutputFile) == 0 {
+		perfConfig.StatsOutputFile = DefaultOutputFile
+	}
+
+	if perfConfig.NumTimesToHitEndpoints == 0 {
+		perfConfig.NumTimesToHitEndpoints = DefaultNumTimesToHitEndpoints
+	}
+
+	return perfConfig
+}
+
 func populateConstructionMissingFields(
 	constructionConfig *ConstructionConfiguration,
 ) *ConstructionConfiguration {
@@ -171,6 +202,7 @@ func populateMissingFields(config *Configuration) *Configuration {
 
 	config.Construction = populateConstructionMissingFields(config.Construction)
 	config.Data = populateDataMissingFields(config.Data)
+	config.Perf = populatePerfMissingFields(config.Perf)
 
 	return config
 }
diff --git a/configuration/types.go b/configuration/types.go
index 1044143..603afbe 100644
--- a/configuration/types.go
+++ b/configuration/types.go
@@ -61,6 +61,14 @@ const (
 	DefaultStatusPort                        = 9090
 	DefaultMaxReorgDepth                     = 100
 
+	// Check Perf Default Configs
+	DefaultStartBlock                             = 100
+	DefaultEndBlock                               = 10000
+	DefaultNumTimesToHitEndpoints                 = 50
+	DefaultOutputFile                             = "./check_perf_stats.json"
+	DefaultBlockEndpointTimeConstraintMs          = 5000
+	DefaultAccountBalanceEndpointTimeConstraintMs = 5000
+
 	// ETH Defaults
 	EthereumIDBlockchain = "Ethereum"
 	EthereumIDNetwork    = "Ropsten"
@@ -227,7 +235,7 @@ type DataConfiguration struct {
 	InactiveReconciliationConcurrency uint64 `json:"inactive_reconciliation_concurrency"`
 
 	// InactiveReconciliationFrequency is the number of blocks to wait between
-	// inactive reconiliations on each account.
+	// inactive reconciliations on each account.
 	InactiveReconciliationFrequency uint64 `json:"inactive_reconciliation_frequency"`
 
 	// LogBlocks is a boolean indicating whether to log processed blocks.
@@ -340,7 +348,7 @@ type DataConfiguration struct {
 }
 
 // Configuration contains all configuration settings for running
-// check:data or check:construction.
+// check:data, check:construction, or check:perf.
 type Configuration struct {
 	// Network is the *types.NetworkIdentifier where transactions should
 	// be constructed and where blocks should be synced to monitor
@@ -433,4 +441,29 @@ type Configuration struct {
 
 	Construction *ConstructionConfiguration `json:"construction"`
 	Data         *DataConfiguration         `json:"data"`
+	Perf         *CheckPerfConfiguration    `json:"perf"`
+}
+
+//********************//
+// Check Perf configs //
+//********************//
+type CheckPerfConfiguration struct {
+
+	// StartBlock is the starting block for running check:perf.
+	// If not provided, then this defaults to 0 (the genesis block)
+	StartBlock int64 `json:"start_block,omitempty"`
+
+	BlockEndpointTimeConstraintMs int64 `json:"block_endpoint_time_constraint_ms"`
+
+	AccountBalanceEndpointTimeConstraintMs int64 `json:"account_balance_endpoint_time_constraint_ms"`
+
+	// EndBlock is the ending block for running check:perf.
+	// Must be provided when running check:perf
+	EndBlock int64 `json:"end_block"`
+
+	// NumTimesToHitEndpoints is the number of times each rosetta-server endpoint will be benchmarked
+	NumTimesToHitEndpoints int `json:"num_times_to_hit_endpoints"`
+
+	// Location to output test results
+	StatsOutputFile string `json:"check_perf_output_dir"`
 }
diff --git a/examples/configuration/default.json b/examples/configuration/default.json
index 11ee99f..df5c68a 100644
--- a/examples/configuration/default.json
+++ b/examples/configuration/default.json
@@ -39,5 +39,6 @@
   "results_output_file": "",
   "pruning_disabled": false,
   "initial_balance_fetch_disabled": false
- }
+ },
+ "perf": null
 }
\ No newline at end of file
diff --git a/pkg/results/perf_results.go b/pkg/results/perf_results.go
new file mode 100644
index 0000000..a0349c6
--- /dev/null
+++ b/pkg/results/perf_results.go
@@ -0,0 +1,157 @@
+// Copyright 2022 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package results
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"strconv"
+	"time"
+
+	"github.com/coinbase/rosetta-cli/configuration"
+	"github.com/coinbase/rosetta-sdk-go/utils"
+	"github.com/olekukonko/tablewriter"
+)
+
+// Output writes *CheckPerfResults to the provided
+// path.
+func (c *CheckPerfStats) Output(path string) {
+	if len(path) > 0 {
+		writeErr := utils.SerializeAndWrite(path, c)
+		if writeErr != nil {
+			log.Printf("%s: unable to save results\n", writeErr.Error())
+		}
+	}
+}
+
+type CheckPerfRawStats struct {
+	BlockEndpointTotalTime          time.Duration
+	BlockEndpointNumErrors          int64
+	AccountBalanceEndpointTotalTime time.Duration
+	AccountBalanceNumErrors         int64
+}
+
+// CheckPerfStats contains interesting stats that
+// are counted while running the check:perf.
+type CheckPerfStats struct {
+	StartBlock                          int64 `json:"start_block"`
+	EndBlock                            int64 `json:"end_block"`
+	NumTimesHitEachEndpoint             int   `json:"num_times_hit_each_endpoint"`
+	AccountBalanceEndpointAverageTimeMs int64 `json:"account_balance_endpoint_average_time_ms"`
+	AccountBalanceEndpointTotalTimeMs   int64 `json:"account_balance_endpoint_total_time_ms"`
+	AccountBalanceEndpointNumErrors     int64 `json:"account_balance_endpoint_num_errors"`
+	BlockEndpointAverageTimeMs          int64 `json:"block_endpoint_average_time_ms"`
+	BlockEndpointTotalTimeMs            int64 `json:"block_endpoint_total_time_ms"`
+	BlockEndpointNumErrors              int64 `json:"block_endpoint_num_errors"`
+}
+
+// Print logs CheckPerfStats to the console.
+func (c *CheckPerfStats) Print() {
+	table := tablewriter.NewWriter(os.Stdout)
+	table.SetRowLine(true)
+	table.SetRowSeparator("-")
+	table.SetHeader([]string{"check:perf Stats", "Description", "Value"})
+	table.Append([]string{"Start Block", "The Starting Block", strconv.FormatInt(c.StartBlock, 10)})
+	table.Append([]string{"End Block", "The Ending Block", strconv.FormatInt(c.EndBlock, 10)})
+	table.Append([]string{"Num Times Each Endpoint", "Number of times that each endpoint was hit", strconv.FormatInt(int64(c.NumTimesHitEachEndpoint), 10)})
+	table.Append(
+		[]string{
+			"/Block Endpoint Total Time",
+			"Total elapsed time taken to fetch all blocks (ms)",
+			strconv.FormatInt(c.BlockEndpointTotalTimeMs, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Block Endpoint Average Time",
+			"Average time taken to fetch each block (ms)",
+			strconv.FormatInt(c.BlockEndpointAverageTimeMs, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Block Endpoint Num Errors",
+			"Total num errors occurred while fetching blocks",
+			strconv.FormatInt(c.BlockEndpointNumErrors, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Account/Balance Endpoint Average Time",
+			"Average time taken to fetch each account balance (ms)",
+			strconv.FormatInt(c.AccountBalanceEndpointAverageTimeMs, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Account/Balance Endpoint Total Time",
+			"Total elapsed time taken to fetch all account balances (ms)",
+			strconv.FormatInt(c.AccountBalanceEndpointTotalTimeMs, 10),
+		},
+	)
+	table.Append(
+		[]string{
+			"/Account/Balance Endpoint Num Errors",
+			"Total num errors occurred while fetching account balances",
+			strconv.FormatInt(c.AccountBalanceEndpointNumErrors, 10),
+		},
+	)
+
+	table.Render()
+}
+
+// ComputeCheckPerfStats returns a populated CheckPerfStats.
+func ComputeCheckPerfStats(
+	config *configuration.CheckPerfConfiguration,
+	rawStats *CheckPerfRawStats,
+) *CheckPerfStats {
+	totalNumEndpointsHit := (config.EndBlock - config.StartBlock) * int64(config.NumTimesToHitEndpoints)
+	stats := &CheckPerfStats{
+		BlockEndpointAverageTimeMs:          rawStats.BlockEndpointTotalTime.Milliseconds() / totalNumEndpointsHit,
+		BlockEndpointTotalTimeMs:            rawStats.BlockEndpointTotalTime.Milliseconds(),
+		BlockEndpointNumErrors:              rawStats.BlockEndpointNumErrors,
+		AccountBalanceEndpointAverageTimeMs: rawStats.AccountBalanceEndpointTotalTime.Milliseconds() / totalNumEndpointsHit,
+		AccountBalanceEndpointTotalTimeMs:   rawStats.AccountBalanceEndpointTotalTime.Milliseconds(),
+		AccountBalanceEndpointNumErrors:     rawStats.AccountBalanceNumErrors,
+		StartBlock:                          config.StartBlock,
+		EndBlock:                            config.EndBlock,
+		NumTimesHitEachEndpoint:             config.NumTimesToHitEndpoints,
+	}
+
+	return stats
+}
+
+// ExitPerf exits check:perf, logs the test results to the console,
+// and to a provided output path.
+func ExitPerf(
+	config *configuration.CheckPerfConfiguration,
+	err error,
+	rawStats *CheckPerfRawStats,
+) error {
+	if err != nil {
+		log.Fatal(fmt.Errorf("Check:Perf Failed!: %w", err))
+	}
+
+	stats := ComputeCheckPerfStats(
+		config,
+		rawStats,
+	)
+
+	stats.Print()
+	stats.Output(config.StatsOutputFile)
+
+	return err
+}


Test patch:
diff --git a/pkg/tester/benchmark_utils.go b/pkg/tester/benchmark_utils.go
new file mode 100644
index 0000000..eb64dbe
--- /dev/null
+++ b/pkg/tester/benchmark_utils.go
@@ -0,0 +1,24 @@
+// Copyright 2022 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tester
+
+import "time"
+
+func timerFactory() func() time.Duration {
+	start := time.Now()
+	return func() time.Duration {
+		return time.Since(start)
+	}
+}
diff --git a/pkg/tester/data_perf.go b/pkg/tester/data_perf.go
new file mode 100644
index 0000000..0cdefc5
--- /dev/null
+++ b/pkg/tester/data_perf.go
@@ -0,0 +1,96 @@
+// Copyright 2022 Coinbase, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package tester
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/coinbase/rosetta-cli/configuration"
+	"github.com/coinbase/rosetta-cli/pkg/results"
+	"github.com/coinbase/rosetta-sdk-go/fetcher"
+	"github.com/coinbase/rosetta-sdk-go/types"
+)
+
+func SetupBenchmarking(config *configuration.Configuration) (*fetcher.Fetcher, func() time.Duration, chan time.Duration) {
+	// Create a new fetcher
+	fetcher := fetcher.New(
+		config.OnlineURL,
+		fetcher.WithMaxRetries(0),
+	)
+	timer := timerFactory()
+	elapsed := make(chan time.Duration, 1)
+	return fetcher, timer, elapsed
+}
+
+// Benchmark the asset issuer's /block endpoint
+func BmarkBlock(ctx context.Context, config *configuration.Configuration, fetcher *fetcher.Fetcher, timer func() time.Duration, elapsed chan time.Duration, rawStats *results.CheckPerfRawStats) error {
+	total_errors := 0
+	go func() {
+		for m := config.Perf.StartBlock; m < config.Perf.EndBlock; m++ {
+			for n := 0; n < config.Perf.NumTimesToHitEndpoints; n++ {
+				partialBlockId := &types.PartialBlockIdentifier{
+					Hash:  nil,
+					Index: &m,
+				}
+				_, err := fetcher.Block(ctx, config.Network, partialBlockId)
+				if err != nil {
+					total_errors++
+				}
+			}
+		}
+		elapsed <- timer()
+	}()
+	select {
+	case <-ctx.Done():
+		return errors.New("/block endpoint benchmarking timed out")
+	case timeTaken := <-elapsed:
+		rawStats.BlockEndpointTotalTime = timeTaken
+		rawStats.BlockEndpointNumErrors = int64(total_errors)
+		return nil
+	}
+}
+
+// Benchmark the asset issuers /account/balance endpoint
+func BmarkAccountBalance(ctx context.Context, config *configuration.Configuration, fetcher *fetcher.Fetcher, timer func() time.Duration, elapsed chan time.Duration, rawStats *results.CheckPerfRawStats) error {
+	total_errors := 0
+	go func() {
+		for m := config.Perf.StartBlock; m < config.Perf.EndBlock; m++ {
+			for n := 0; n < config.Perf.NumTimesToHitEndpoints; n++ {
+				account := &types.AccountIdentifier{
+					Address: "address",
+				}
+				partialBlockId := &types.PartialBlockIdentifier{
+					Hash:  nil,
+					Index: &m,
+				}
+				_, _, _, err := fetcher.AccountBalance(ctx, config.Network, account, partialBlockId, nil)
+				if err != nil {
+					total_errors++
+				}
+			}
+		}
+		elapsed <- timer()
+	}()
+	select {
+	case <-ctx.Done():
+		return errors.New("/account/balance endpoint benchmarking timed out")
+	case timeTaken := <-elapsed:
+		rawStats.AccountBalanceEndpointTotalTime = timeTaken
+		rawStats.AccountBalanceNumErrors = int64(total_errors)
+		return nil
+	}
+}