Repository: oasisprotocol/oasis-core (go)

Issue:
Update storage backend to Badger v3
We should migrate the storage db backend to [Badger v3](https://pkg.go.dev/github.com/dgraph-io/badger/v3) (currently we are using v2). This is a breaking change requiring all (internal) key-value pairs to be migrated (possibly using Badger's streaming backup support).

Pull request:
go: Bump Badger to v3
Fixes #3859 

TODO
* [x] A couple of long-term test runs.
* [x] Migration (possibly using the Badger backup/restore system) from badger v2 as the versions are incompatible.

Patch:
diff --git a/.changelog/3859.feature.md b/.changelog/3859.feature.md
new file mode 100644
index 000000000..59890a50d
--- /dev/null
+++ b/.changelog/3859.feature.md
@@ -0,0 +1,15 @@
+Bump Badger database to v3
+
+The old database will automatically be converted during the initial node
+startup. For this reason the following should be taken into account when
+upgrading:
+
+- Initial startup time will be longer as a full database migration needs to
+  be performed for each of the internal databases. The migration may take up
+  to an hour for large databases.
+
+- In order to ensure a backup exists in the unlikely case that anything goes
+  wrong with the migration, the old databases will be kept in the same
+  location but with a `.backup` suffix. *For this reason, make sure that you
+  have enough free disk space to accommodate a copy of the node's data
+  directory.*
diff --git a/go/common/badger/helpers.go b/go/common/badger/helpers.go
index d5d9714f6..4d1d8e759 100644
--- a/go/common/badger/helpers.go
+++ b/go/common/badger/helpers.go
@@ -2,12 +2,17 @@
 package badger
 
 import (
+	"bufio"
 	"fmt"
+	"io"
+	"math"
+	"os"
 	"strings"
 	"sync"
 	"time"
 
-	"github.com/dgraph-io/badger/v2"
+	badgerV2 "github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 
 	"github.com/oasisprotocol/oasis-core/go/common/logging"
 )
@@ -15,6 +20,10 @@ import (
 const (
 	gcInterval     = 5 * time.Minute
 	gcDiscardRatio = 0.5
+
+	migrationBufferSize      = 64 << 20 // 64 MiB
+	migrationTemporarySuffix = ".migration"
+	migrationBackupSuffix    = ".backup"
 )
 
 // NewLogAdapter returns a badger.Logger backed by an oasis-node logger.
@@ -110,3 +119,125 @@ func NewGCWorker(logger *logging.Logger, db *badger.DB) *GCWorker {
 
 	return gc
 }
+
+// Open returns a new DB object.
+func Open(opts badger.Options) (*badger.DB, error) {
+	return openWithMigrations(opts, false)
+}
+
+// OpenManaged returns a new DB object in managed mode.
+func OpenManaged(opts badger.Options) (*badger.DB, error) {
+	return openWithMigrations(opts, true)
+}
+
+func openWithMigrations(opts badger.Options, managed bool) (*badger.DB, error) {
+	openFn := badger.Open
+	if managed {
+		openFn = badger.OpenManaged
+	}
+
+	db, err := openFn(opts)
+	if err == nil {
+		return db, nil
+	}
+
+	// Check if the error indicates that a migration is needed.
+	// XXX: Since badger does not support indicating this via an exported error type, we need
+	//      to resort to string matching which is not ideal.
+	if !strings.Contains(err.Error(), "manifest has unsupported version") {
+		return nil, err
+	}
+
+	// Perform the migration.
+	if err := migrateDatabase(opts); err != nil {
+		return nil, fmt.Errorf("migration failed: %w", err)
+	}
+
+	// Retry opening the database.
+	return openFn(opts)
+}
+
+func migrateDatabase(opts badger.Options) error {
+	var logger *logging.Logger
+	adapter, _ := opts.Logger.(*badgerLogger)
+	if logger != nil {
+		logger = adapter.logger
+	} else {
+		logger = logging.GetLogger("common/badger")
+	}
+	logger = logger.With("db", opts.Dir)
+
+	logger.Warn("performing a database migration")
+
+	// We don't use such a configuration anywhere, but make sure anyway.
+	if opts.ValueDir != opts.Dir {
+		return fmt.Errorf("migrations with separate value directory not yet supported")
+	}
+
+	// Remove any leftovers.
+	temporaryDbName := opts.Dir + migrationTemporarySuffix
+	if err := os.RemoveAll(temporaryDbName); err != nil {
+		return fmt.Errorf("failed to remove temporary destination '%s': %w", temporaryDbName, err)
+	}
+
+	// Open the database as Badger v2.
+	optsV2 := badgerV2.DefaultOptions(opts.Dir)
+	optsV2 = optsV2.WithNumVersionsToKeep(math.MaxInt32)
+	optsV2 = optsV2.WithLogger(nil)
+
+	dbV2, err := badgerV2.Open(optsV2)
+	if err != nil {
+		return fmt.Errorf("failed to open source database: %w", err)
+	}
+	defer dbV2.Close()
+
+	// Open the destination database as Badger v3.
+	optsV3 := badger.DefaultOptions(temporaryDbName)
+	optsV3 = optsV3.WithNumVersionsToKeep(math.MaxInt32)
+	optsV3 = optsV3.WithLogger(NewLogAdapter(logger))
+
+	dbV3, err := badger.Open(optsV3)
+	if err != nil {
+		return fmt.Errorf("failed to open destination database: %w", err)
+	}
+	defer dbV3.Close()
+
+	r, w := io.Pipe()
+
+	// Start the backup goroutine.
+	backupCh := make(chan error, 1)
+	go func() {
+		defer close(backupCh)
+
+		bw := bufio.NewWriterSize(w, migrationBufferSize)
+		defer w.Close()
+		defer bw.Flush()
+
+		_, errBackup := dbV2.Backup(bw, 0)
+		backupCh <- errBackup
+	}()
+
+	// Start the restore process.
+	if err := dbV3.Load(r, 256); err != nil {
+		return fmt.Errorf("failed to restore backup: %w", err)
+	}
+	if err := <-backupCh; err != nil {
+		return fmt.Errorf("failed to create backup: %w", err)
+	}
+
+	// Close both databases.
+	dbV2.Close()
+	dbV3.Close()
+
+	// Rename old database to keep as a backup and rename new database in place of the old one.
+	if err := os.Rename(opts.Dir, opts.Dir+migrationBackupSuffix); err != nil {
+		return fmt.Errorf("failed to rename source database to '%s': %w", opts.Dir+migrationBackupSuffix, err)
+	}
+	if err := os.Rename(opts.Dir+migrationTemporarySuffix, opts.Dir); err != nil {
+		return fmt.Errorf("failed to rename destination database to '%s': %w", opts.Dir, err)
+	}
+
+	logger.Warn("database successfully migrated")
+
+	return nil
+}
diff --git a/go/common/persistent/persistent.go b/go/common/persistent/persistent.go
index b01ce15ae..d6f345ac3 100644
--- a/go/common/persistent/persistent.go
+++ b/go/common/persistent/persistent.go
@@ -8,8 +8,8 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
+	"github.com/dgraph-io/badger/v3"
+	"github.com/dgraph-io/badger/v3/options"
 
 	cmnBadger "github.com/oasisprotocol/oasis-core/go/common/badger"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
@@ -55,12 +55,9 @@ func NewCommonStore(dataDir string) (*CommonStore, error) {
 	opts := badger.DefaultOptions(GetPersistentStoreDBDir(dataDir))
 	opts = opts.WithLogger(cmnBadger.NewLogAdapter(logger))
 	opts = opts.WithSyncWrites(true)
-	// Allow value log truncation if required (this is needed to recover the
-	// value log file which can get corrupted in crashes).
-	opts = opts.WithTruncate(true)
 	opts = opts.WithCompression(options.None)
 
-	db, err := badger.Open(opts)
+	db, err := cmnBadger.Open(opts)
 	if err != nil {
 		return nil, fmt.Errorf("failed to open persistence database: %w", err)
 	}
diff --git a/go/consensus/tendermint/db/badger/badger.go b/go/consensus/tendermint/db/badger/badger.go
index 040089704..9fc154d28 100644
--- a/go/consensus/tendermint/db/badger/badger.go
+++ b/go/consensus/tendermint/db/badger/badger.go
@@ -9,8 +9,8 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
+	"github.com/dgraph-io/badger/v3"
+	"github.com/dgraph-io/badger/v3/options"
 	"github.com/tendermint/tendermint/node"
 	dbm "github.com/tendermint/tm-db"
 
@@ -66,13 +66,10 @@ func New(fn string, noSuffix bool) (dbm.DB, error) {
 	opts := badger.DefaultOptions(fn) // This may benefit from LSMOnlyOptions.
 	opts = opts.WithLogger(cmnBadger.NewLogAdapter(logger))
 	opts = opts.WithSyncWrites(false)
-	// Allow value log truncation if required (this is needed to recover the
-	// value log file which can get corrupted in crashes).
-	opts = opts.WithTruncate(true)
 	opts = opts.WithCompression(options.Snappy)
 	opts = opts.WithBlockCacheSize(64 * 1024 * 1024)
 
-	db, err := badger.Open(opts)
+	db, err := cmnBadger.Open(opts)
 	if err != nil {
 		return nil, fmt.Errorf("tendermint/db/badger: failed to open database: %w", err)
 	}
diff --git a/go/go.mod b/go/go.mod
index 5cf504e85..ca6107b75 100644
--- a/go/go.mod
+++ b/go/go.mod
@@ -22,6 +22,7 @@ require (
 	github.com/btcsuite/btcutil v1.0.2
 	github.com/cenkalti/backoff/v4 v4.1.0
 	github.com/dgraph-io/badger/v2 v2.2007.2
+	github.com/dgraph-io/badger/v3 v3.2011.1
 	github.com/eapache/channels v1.1.0
 	github.com/fxamacker/cbor/v2 v2.2.1-0.20200820021930-bafca87fa6db
 	github.com/go-kit/kit v0.10.0
diff --git a/go/go.sum b/go/go.sum
index 571a98aa8..f7e869a3a 100644
--- a/go/go.sum
+++ b/go/go.sum
@@ -143,6 +143,7 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
 github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cosiner/argv v0.1.0/go.mod h1:EusR6TucWKX+zFgtdUsKT2Cvg45K5rtpCcWz4hK06d8=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d h1:49RLWk1j44Xu4fjHb6JFYmeUnDORVwHNkDxaQ0ctCVU=
 github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y=
 github.com/cosmos/iavl v0.15.3 h1:xE9r6HW8GeKeoYJN4zefpljZ1oukVScP/7M8oj6SUts=
@@ -155,6 +156,7 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d h1:SwD98825d6bdB+pEuTxWOXiSjBrHdOl/UVp75eI7JT8=
 github.com/cznic/b v0.0.0-20181122101859-a26611c4d92d/go.mod h1:URriBxXwVq5ijiJ12C7iIZqlA69nTlI+LgI6/pwftG8=
 github.com/cznic/mathutil v0.0.0-20181122101859-297441e03548/go.mod h1:e6NPNENfs9mPDVNRekM7lKScauxd5kXTr1Mfyig6TDM=
@@ -174,9 +176,12 @@ github.com/dgraph-io/badger v1.6.1 h1:w9pSFNSdq/JPM1N12Fz/F/bzo993Is1W+Q7HjPzi7y
 github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU=
 github.com/dgraph-io/badger/v2 v2.2007.2 h1:EjjK0KqwaFMlPin1ajhP943VPENHJdEz1KLIegjaI3k=
 github.com/dgraph-io/badger/v2 v2.2007.2/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE=
+github.com/dgraph-io/badger/v3 v3.2011.1 h1:Hmyof0WMEF/QtutX5SQHzIMnJQxb/IrSzhjckV2SD6g=
+github.com/dgraph-io/badger/v3 v3.2011.1/go.mod h1:0rLLrQpKVQAL0or/lBLMQznhr6dWWX7h5AKnmnqx268=
 github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
-github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de h1:t0UHb5vdojIDUqktM6+xJAfScFBsVpXZmqC9dsgJmeA=
 github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E=
+github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d h1:eQYOG6A4td1tht0NdJB9Ls6DsXRGb2Ft6X9REU/MbbE=
+github.com/dgraph-io/ristretto v0.0.4-0.20210122082011-bb5d392ed82d/go.mod h1:tv2ec8nA7vRpSYX7/MbP52ihrUMXIHit54CQMq8npXQ=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
 github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
@@ -230,6 +235,7 @@ github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod
 github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31 h1:gclg6gY70GLy3PbkQ1AERPfmLMMagS60DKF78eWwLn8=
 github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24=
 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
+github.com/go-delve/delve v1.5.0/go.mod h1:c6b3a1Gry6x8a4LGCe/CWzrocrfaHvkUxCj3k4bvSUQ=
 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -290,6 +296,8 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/flatbuffers v1.12.0 h1:/PtAHvnBY4Kqnx/xCQ3OIV9uYcSFGScBsWI3Oogeh6w=
+github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
@@ -300,6 +308,7 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-dap v0.2.0/go.mod h1:5q8aYQFnHOAZEMP+6vmq25HKYAEwE+LF5yh7JKrrhSQ=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
@@ -697,6 +706,7 @@ github.com/marten-seemann/qtls v0.10.0 h1:ECsuYUKalRL240rRD4Ri33ISb7kAQ3qGDlrrl5
 github.com/marten-seemann/qtls v0.10.0/go.mod h1:UvMd1oaYDACI99/oZUYLzMCkBXQVT0aGm99sJhbT8hs=
 github.com/marten-seemann/qtls-go1-15 v0.1.1 h1:LIH6K34bPVttyXnUWixk0bzH6/N07VxbSabxn5A5gZQ=
 github.com/marten-seemann/qtls-go1-15 v0.1.1/go.mod h1:GyFwywLKkRt+6mfU99csTEY1joMZz5vmB1WNZH3P81I=
+github.com/mattn/go-colorable v0.0.0-20170327083344-ded68f7a9561/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-colorable v0.1.4 h1:snbPLB8fVfU9iwbbo30TPtbLRzwWu6aJS6Xh4eaaviA=
@@ -740,6 +750,7 @@ github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0Qu
 github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mmcloughlin/avo v0.0.0-20201105074841-5d2f697d268f/go.mod h1:6aKT4zZIrpGqB3RpFU14ByCSSyKY6LfJz4J/JJChHfI=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -869,6 +880,7 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP
 github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
 github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac=
+github.com/peterh/liner v0.0.0-20170317030525-88609521dc4b/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ=
 github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o=
 github.com/philhofer/fwd v1.0.0 h1:UbZqGr5Y38ApvM/V/jEljVxwocdweyH+vmYvRPBnbqQ=
@@ -988,12 +1000,14 @@ github.com/spf13/afero v1.1.2 h1:m8/z1t7/fwjysjQRYbP0RD+bUIF/8tJwPdEZsI83ACI=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.0-20170417170307-b6cb39589372/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/cobra v1.1.1 h1:KfztREH0tPxJJ+geloSLaAkaPkr4ki2Er5quFV1TDo4=
 github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v0.0.0-20170417173400-9e4c21054fa1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
@@ -1036,6 +1050,7 @@ github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU=
 github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/twitchyliquid64/golang-asm v0.15.0/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/uber/jaeger-client-go v2.25.0+incompatible h1:IxcNZ7WRY1Y3G4poYlx24szfsn/3LvK9QHCq9oQw8+U=
 github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
 github.com/uber/jaeger-lib v2.2.0+incompatible h1:MxZXOiR2JuoANZ3J6DE/U0kSFv/eJ/GfSYVCjK7dyaw=
@@ -1093,8 +1108,10 @@ go.opencensus.io v0.22.1/go.mod h1:Ap50jQcDJrx6rB6VgeeFPtuPIf3wMRvRfrfYDO6+BmA=
 go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
 go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M=
 go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E=
+go.starlark.net v0.0.0-20190702223751-32f345186213/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=
@@ -1116,6 +1133,8 @@ go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc=
 go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM=
 go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ=
 go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE=
+golang.org/x/arch v0.0.0-20190927153633-4e8777c89be4/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
+golang.org/x/arch v0.0.0-20201008161808-52c3e6f60cff/go.mod h1:flIaEI6LNU6xOCD5PaJvn9wGP0agmIOqjrtsKGRguv4=
 golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw=
 golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -1328,11 +1347,13 @@ golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191127201027-ecd32218bd7f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20201105001634-bc3cf281b174/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a h1:CB3a9Nez8M13wwlr/E2YtwoU+qYHKfC+JrDa45RXXoQ=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1461,6 +1482,7 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.1-2019.2.3 h1:3JgtbtFHMiCmsznwGVTUWbgGov+pVqnlf1dEJTNAXeM=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
 rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
 sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
diff --git a/go/runtime/history/db.go b/go/runtime/history/db.go
index 3ac05629d..76280efb0 100644
--- a/go/runtime/history/db.go
+++ b/go/runtime/history/db.go
@@ -3,8 +3,8 @@ package history
 import (
 	"fmt"
 
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
+	"github.com/dgraph-io/badger/v3"
+	"github.com/dgraph-io/badger/v3/options"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
 	cmnBadger "github.com/oasisprotocol/oasis-core/go/common/badger"
@@ -57,12 +57,9 @@ func newDB(fn string, runtimeID common.Namespace) (*DB, error) {
 	opts := badger.DefaultOptions(fn)
 	opts = opts.WithLogger(cmnBadger.NewLogAdapter(logger))
 	opts = opts.WithSyncWrites(true)
-	// Allow value log truncation if required (this is needed to recover the
-	// value log file which can get corrupted in crashes).
-	opts = opts.WithTruncate(true)
 	opts = opts.WithCompression(options.None)
 
-	db, err := badger.Open(opts)
+	db, err := cmnBadger.Open(opts)
 	if err != nil {
 		return nil, fmt.Errorf("runtime/history: failed to open database: %w", err)
 	}
diff --git a/go/runtime/history/prune.go b/go/runtime/history/prune.go
index 6a8ade245..81c245768 100644
--- a/go/runtime/history/prune.go
+++ b/go/runtime/history/prune.go
@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 
 	"github.com/oasisprotocol/oasis-core/go/common/logging"
 )
diff --git a/go/runtime/localstorage/localstorage.go b/go/runtime/localstorage/localstorage.go
index ef3332eec..e623fa2dd 100644
--- a/go/runtime/localstorage/localstorage.go
+++ b/go/runtime/localstorage/localstorage.go
@@ -8,8 +8,8 @@ import (
 	"fmt"
 	"path/filepath"
 
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
+	"github.com/dgraph-io/badger/v3"
+	"github.com/dgraph-io/badger/v3/options"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
 	cmnBadger "github.com/oasisprotocol/oasis-core/go/common/badger"
@@ -111,13 +111,10 @@ func New(dataDir, fn string, runtimeID common.Namespace) (LocalStorage, error) {
 	opts := badger.DefaultOptions(filepath.Join(dataDir, fn))
 	opts = opts.WithLogger(cmnBadger.NewLogAdapter(s.logger))
 	opts = opts.WithSyncWrites(true)
-	// Allow value log truncation if required (this is needed to recover the
-	// value log file which can get corrupted in crashes).
-	opts = opts.WithTruncate(true)
 	opts = opts.WithCompression(options.None)
 
 	var err error
-	if s.db, err = badger.Open(opts); err != nil {
+	if s.db, err = cmnBadger.Open(opts); err != nil {
 		return nil, fmt.Errorf("failed to open local storage database: %w", err)
 	}
 	s.gc = cmnBadger.NewGCWorker(s.logger, s.db)
diff --git a/go/storage/mkvs/db/badger/badger.go b/go/storage/mkvs/db/badger/badger.go
index dddbb7d09..c89bf1fc2 100644
--- a/go/storage/mkvs/db/badger/badger.go
+++ b/go/storage/mkvs/db/badger/badger.go
@@ -7,7 +7,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
 	cmnBadger "github.com/oasisprotocol/oasis-core/go/common/badger"
@@ -76,7 +76,7 @@ func New(cfg *api.Config) (api.NodeDB, error) {
 	opts := commonConfigToBadgerOptions(cfg, db)
 
 	var err error
-	if db.db, err = badger.OpenManaged(opts); err != nil {
+	if db.db, err = cmnBadger.OpenManaged(opts); err != nil {
 		return nil, fmt.Errorf("mkvs/badger: failed to open database: %w", err)
 	}
 
diff --git a/go/storage/mkvs/db/badger/check.go b/go/storage/mkvs/db/badger/check.go
index f480d02ca..0ba510c5b 100644
--- a/go/storage/mkvs/db/badger/check.go
+++ b/go/storage/mkvs/db/badger/check.go
@@ -5,8 +5,9 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 
+	cmnBadger "github.com/oasisprotocol/oasis-core/go/common/badger"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
 	"github.com/oasisprotocol/oasis-core/go/common/crypto/hash"
 	"github.com/oasisprotocol/oasis-core/go/common/logging"
@@ -216,7 +217,7 @@ func CheckSanity(ctx context.Context, cfg *api.Config, display DisplayHelper) er
 	opts := commonConfigToBadgerOptions(&roCfg, db)
 
 	var err error
-	if db.db, err = badger.OpenManaged(opts); err != nil {
+	if db.db, err = cmnBadger.OpenManaged(opts); err != nil {
 		return fmt.Errorf("mkvs/badger/check: failed to open database: %w", err)
 	}
 	defer db.Close()
diff --git a/go/storage/mkvs/db/badger/helpers.go b/go/storage/mkvs/db/badger/helpers.go
index e97f5dfeb..195eec103 100644
--- a/go/storage/mkvs/db/badger/helpers.go
+++ b/go/storage/mkvs/db/badger/helpers.go
@@ -1,8 +1,8 @@
 package badger
 
 import (
-	"github.com/dgraph-io/badger/v2"
-	"github.com/dgraph-io/badger/v2/options"
+	"github.com/dgraph-io/badger/v3"
+	"github.com/dgraph-io/badger/v3/options"
 
 	cmnBadger "github.com/oasisprotocol/oasis-core/go/common/badger"
 	"github.com/oasisprotocol/oasis-core/go/storage/mkvs/db/api"
@@ -31,11 +31,13 @@ func commonConfigToBadgerOptions(cfg *api.Config, db *badgerNodeDB) badger.Optio
 	opts := badger.DefaultOptions(cfg.DB)
 	opts = opts.WithLogger(cmnBadger.NewLogAdapter(db.logger))
 	opts = opts.WithSyncWrites(!cfg.NoFsync)
-	// Allow value log truncation if required (this is needed to recover the
-	// value log file which can get corrupted in crashes).
-	opts = opts.WithTruncate(true)
 	opts = opts.WithCompression(options.Snappy)
-	opts = opts.WithBlockCacheSize(cfg.MaxCacheSize)
+	if cfg.MaxCacheSize == 0 {
+		// Default to 64mb block cache size if not configured to avoid a panic.
+		opts = opts.WithBlockCacheSize(64 * 1024 * 1024)
+	} else {
+		opts = opts.WithBlockCacheSize(cfg.MaxCacheSize)
+	}
 	opts = opts.WithReadOnly(cfg.ReadOnly)
 	opts = opts.WithDetectConflicts(false)
 
diff --git a/go/storage/mkvs/db/badger/metadata.go b/go/storage/mkvs/db/badger/metadata.go
index a9e8bbba2..e265891eb 100644
--- a/go/storage/mkvs/db/badger/metadata.go
+++ b/go/storage/mkvs/db/badger/metadata.go
@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"sync"
 
-	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
diff --git a/go/storage/mkvs/db/badger/migrate.go b/go/storage/mkvs/db/badger/migrate.go
index 5ffd49c14..5d8e7bbc1 100644
--- a/go/storage/mkvs/db/badger/migrate.go
+++ b/go/storage/mkvs/db/badger/migrate.go
@@ -6,8 +6,9 @@ import (
 	"fmt"
 	"math"
 
-	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 
+	cmnBadger "github.com/oasisprotocol/oasis-core/go/common/badger"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
 	"github.com/oasisprotocol/oasis-core/go/common/crypto/hash"
 	"github.com/oasisprotocol/oasis-core/go/common/keyformat"
@@ -1389,7 +1390,7 @@ func Migrate(cfg *api.Config, helper MigrationHelper) (uint64, error) {
 	opts := commonConfigToBadgerOptions(cfg, db)
 
 	var err error
-	if db.db, err = badger.OpenManaged(opts); err != nil {
+	if db.db, err = cmnBadger.OpenManaged(opts); err != nil {
 		return 0, fmt.Errorf("mkvs/badger/migrate: failed to open database: %w", err)
 	}
 	defer db.Close()
diff --git a/go/storage/mkvs/db/badger/rename.go b/go/storage/mkvs/db/badger/rename.go
index fd463aa93..de4deac94 100644
--- a/go/storage/mkvs/db/badger/rename.go
+++ b/go/storage/mkvs/db/badger/rename.go
@@ -3,9 +3,10 @@ package badger
 import (
 	"fmt"
 
-	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
+	cmnBadger "github.com/oasisprotocol/oasis-core/go/common/badger"
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"
 	"github.com/oasisprotocol/oasis-core/go/common/logging"
 	"github.com/oasisprotocol/oasis-core/go/storage/mkvs/db/api"
@@ -21,7 +22,7 @@ func RenameNamespace(cfg *api.Config, newNamespace common.Namespace) error {
 	opts := commonConfigToBadgerOptions(cfg, db)
 
 	var err error
-	if db.db, err = badger.OpenManaged(opts); err != nil {
+	if db.db, err = cmnBadger.OpenManaged(opts); err != nil {
 		return fmt.Errorf("failed to open database: %w", err)
 	}
 	defer db.Close()


Test patch:
diff --git a/go/storage/mkvs/db/badger/badger_test.go b/go/storage/mkvs/db/badger/badger_test.go
index e9ecb1259..6fe332b90 100644
--- a/go/storage/mkvs/db/badger/badger_test.go
+++ b/go/storage/mkvs/db/badger/badger_test.go
@@ -9,7 +9,7 @@ import (
 	"strconv"
 	"testing"
 
-	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 	"github.com/stretchr/testify/require"
 
 	"github.com/oasisprotocol/oasis-core/go/common"
diff --git a/go/storage/mkvs/db/badger/migrate_test.go b/go/storage/mkvs/db/badger/migrate_test.go
index dd2d88a5f..c07c72831 100644
--- a/go/storage/mkvs/db/badger/migrate_test.go
+++ b/go/storage/mkvs/db/badger/migrate_test.go
@@ -10,7 +10,7 @@ import (
 	"path/filepath"
 	"testing"
 
-	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v3"
 	"github.com/stretchr/testify/require"
 
 	"github.com/oasisprotocol/oasis-core/go/common/cbor"