Repository: coinbase/rosetta-cli
Pull request #166 (2020-10-15 18:47:22)

Dynamic Pruning Support
This PR adds support for new performance options meant to vastly improve syncing speed on the `check:data` command.

### Changes
- [x] Add dynamic pruning (to support reconciliation backlog)
- [x] Add new optional performance options (`compression_disabled` and `memory_limit_disabled`)
- [x] Update to [`rosetta-sdk-go@v0.5.4`](https://github.com/coinbase/rosetta-sdk-go/releases/tag/v0.5.4)


Patch:
diff --git a/configuration/types.go b/configuration/types.go
index d0ec97c..90c7903 100644
--- a/configuration/types.go
+++ b/configuration/types.go
@@ -341,6 +341,17 @@ type Configuration struct {
 	// should be printed to the console when a file is loaded.
 	LogConfiguration bool `json:"log_configuration"`
 
+	// CompressionDisabled configures the storage layer to not
+	// perform data compression before writing to disk. This leads
+	// to significantly more on-disk storage usage but can lead
+	// to performance gains.
+	CompressionDisabled bool `json:"compression_disabled"`
+
+	// MemoryLimitDisabled configures storage to increase memory
+	// usage. Enabling this massively increases performance
+	// but can use 10s of GBs of RAM, even with pruning enabled.
+	MemoryLimitDisabled bool `json:"memory_limit_disabled"`
+
 	Construction *ConstructionConfiguration `json:"construction"`
 	Data         *DataConfiguration         `json:"data"`
 }
diff --git a/examples/configuration/default.json b/examples/configuration/default.json
index 46869fe..91b8218 100644
--- a/examples/configuration/default.json
+++ b/examples/configuration/default.json
@@ -12,6 +12,8 @@
  "max_sync_concurrency": 64,
  "tip_delay": 300,
  "log_configuration": false,
+ "compression_disabled": false,
+ "memory_limit_disabled": false,
  "construction": null,
  "data": {
   "active_reconciliation_concurrency": 16,
diff --git a/go.mod b/go.mod
index be91211..82218dc 100644
--- a/go.mod
+++ b/go.mod
@@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli
 go 1.13
 
 require (
-	github.com/coinbase/rosetta-sdk-go v0.5.3
+	github.com/coinbase/rosetta-sdk-go v0.5.4
 	github.com/fatih/color v1.9.0
 	github.com/olekukonko/tablewriter v0.0.2-0.20190409134802-7e037d187b0c
 	github.com/spf13/cobra v1.1.0
diff --git a/go.sum b/go.sum
index 7e847b3..07c43ab 100644
--- a/go.sum
+++ b/go.sum
@@ -84,6 +84,10 @@ github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3 h1:p8p0d
 github.com/coinbase/rosetta-sdk-go v0.5.3-0.20201015002757-978c228143a3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws=
 github.com/coinbase/rosetta-sdk-go v0.5.3 h1:AujxZ2VDbq80x7SJDHd1nKOYA8AFD5H27gx07uerFpw=
 github.com/coinbase/rosetta-sdk-go v0.5.3/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws=
+github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51 h1:hcSMD3TDZF4VMJOU2395L60CFgR8JOdtIT4vb6B97nQ=
+github.com/coinbase/rosetta-sdk-go v0.5.4-0.20201015182847-250f7d0adf51/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws=
+github.com/coinbase/rosetta-sdk-go v0.5.4 h1:pM18LK2ci8zZwIu+uETmP6BXHqZbQGXRulwQCjYudg4=
+github.com/coinbase/rosetta-sdk-go v0.5.4/go.mod h1:QVVeKHWFNb0NyzEY06LxXMAylJkYa7n+Hk03pORr0ws=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
diff --git a/pkg/results/construction_results.go b/pkg/results/construction_results.go
index 04ec06f..83e4413 100644
--- a/pkg/results/construction_results.go
+++ b/pkg/results/construction_results.go
@@ -51,8 +51,10 @@ func (c *CheckConstructionResults) Print() {
 	}
 
 	fmt.Printf("\n")
-	c.Stats.Print()
-	fmt.Printf("\n")
+	if c.Stats != nil {
+		c.Stats.Print()
+		fmt.Printf("\n")
+	}
 }
 
 // Output writes CheckConstructionResults to the provided


Test patch:
diff --git a/pkg/tester/construction.go b/pkg/tester/construction.go
index 50fff81..93c8df7 100644
--- a/pkg/tester/construction.go
+++ b/pkg/tester/construction.go
@@ -84,7 +84,15 @@ func InitializeConstruction(
 		log.Fatalf("%s: cannot create command path", err.Error())
 	}
 
-	localStore, err := storage.NewBadgerStorage(ctx, dataPath)
+	opts := []storage.BadgerOption{}
+	if config.CompressionDisabled {
+		opts = append(opts, storage.WithoutCompression())
+	}
+	if config.MemoryLimitDisabled {
+		opts = append(opts, storage.WithCustomSettings(storage.PerformanceBadgerOptions(dataPath)))
+	}
+
+	localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...)
 	if err != nil {
 		log.Fatalf("%s: unable to initialize database", err.Error())
 	}
diff --git a/pkg/tester/data.go b/pkg/tester/data.go
index dbbf05a..4bdb1d1 100644
--- a/pkg/tester/data.go
+++ b/pkg/tester/data.go
@@ -64,7 +64,8 @@ const (
 	EndAtTipCheckInterval = 10 * time.Second
 )
 
-var _ http.Handler = (*ConstructionTester)(nil)
+var _ http.Handler = (*DataTester)(nil)
+var _ statefulsyncer.PruneHelper = (*DataTester)(nil)
 
 // DataTester coordinates the `check:data` test.
 type DataTester struct {
@@ -145,7 +146,15 @@ func InitializeData(
 		log.Fatalf("%s: cannot create command path", err.Error())
 	}
 
-	localStore, err := storage.NewBadgerStorage(ctx, dataPath)
+	opts := []storage.BadgerOption{}
+	if config.CompressionDisabled {
+		opts = append(opts, storage.WithoutCompression())
+	}
+	if config.MemoryLimitDisabled {
+		opts = append(opts, storage.WithCustomSettings(storage.PerformanceBadgerOptions(dataPath)))
+	}
+
+	localStore, err := storage.NewBadgerStorage(ctx, dataPath, opts...)
 	if err != nil {
 		log.Fatalf("%s: unable to initialize database", err.Error())
 	}
@@ -325,7 +334,21 @@ func (t *DataTester) StartPruning(
 		return nil
 	}
 
-	return t.syncer.Prune(ctx, statefulsyncer.DefaultPruningDepth)
+	return t.syncer.Prune(ctx, t)
+}
+
+// PruneableIndex is the index that is
+// safe for pruning.
+func (t *DataTester) PruneableIndex(
+	ctx context.Context,
+	headIndex int64,
+) (int64, error) {
+	if t.config.Data.ReconciliationDisabled {
+		// It is ok if the returned value here is negative
+		return headIndex - statefulsyncer.DefaultPruningDepth, nil
+	}
+
+	return t.reconciler.LastIndexReconciled() - 1, nil
 }
 
 // StartReconciler starts the reconciler if