Repository: dgraph-io/badger
Pull request #1566 (2020-10-14 11:52:28)

Use opencensus for tracing
This PR replaces the `/net/trace` package with `opencensus` for tracing.

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/dgraph-io/badger/1566)
<!-- Reviewable:end -->



Patch:
diff --git a/badger/main.go b/badger/main.go
index 43fe27c..c284f5a 100644
--- a/badger/main.go
+++ b/badger/main.go
@@ -25,6 +25,7 @@ import (
 	"github.com/dgraph-io/badger/v2/badger/cmd"
 	"github.com/dgraph-io/ristretto/z"
 	"github.com/dustin/go-humanize"
+	"go.opencensus.io/zpages"
 )
 
 func main() {
@@ -38,6 +39,7 @@ func main() {
 			}
 		}
 	}()
+	zpages.Handle(nil, "/z")
 	runtime.SetBlockProfileRate(100)
 	runtime.GOMAXPROCS(128)
 
diff --git a/discard.go b/discard.go
index 7b2d3d5..ba15191 100644
--- a/discard.go
+++ b/discard.go
@@ -41,7 +41,6 @@ const discardFname string = "DISCARD"
 const discardFsize int = 1 << 30
 const maxSlot int = 64 << 20
 
-// TODO(naman): Add a test for this.
 func initDiscardStats(opt Options) (*discardStats, error) {
 	fname := path.Join(opt.ValueDir, discardFname)
 
diff --git a/go.mod b/go.mod
index 3208b4b..d3bf1ca 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ require (
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
 	github.com/spf13/cobra v0.0.5
 	github.com/stretchr/testify v1.4.0
+	go.opencensus.io v0.22.5
 	golang.org/x/net v0.0.0-20190620200207-3b0461eec859
 	golang.org/x/sys v0.0.0-20200918174421-af09f7315aff
 	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
diff --git a/go.sum b/go.sum
index c64123e..3fde55c 100644
--- a/go.sum
+++ b/go.sum
@@ -1,3 +1,4 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
 github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
@@ -6,6 +7,7 @@ github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAE
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
@@ -13,8 +15,6 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgraph-io/ristretto v0.0.4-0.20201012224315-0af15dd47cb9 h1:/E/ew7/iVTZnsWhUGi+dNpcWl+4D6FfojPmqX7nghWI=
-github.com/dgraph-io/ristretto v0.0.4-0.20201012224315-0af15dd47cb9/go.mod h1:bDI4cDaalvYSji3vBVDKrn9ouDZrwN974u8ZO/AhYXs=
 github.com/dgraph-io/ristretto v0.0.4-0.20201013194302-6d6fac64beae h1:yh5085twGpsgfuu56DXKOM3SKyZKQPskJIoMNb3jzos=
 github.com/dgraph-io/ristretto v0.0.4-0.20201013194302-6d6fac64beae/go.mod h1:bDI4cDaalvYSji3vBVDKrn9ouDZrwN974u8ZO/AhYXs=
 github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA=
@@ -22,12 +22,20 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn
 github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 h1:ZgQEtGgCBiWRM39fZuwSd1LwSqqSW0hOdXCYYDX0R3I=
+github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/flatbuffers v1.12.0 h1:/PtAHvnBY4Kqnx/xCQ3OIV9uYcSFGScBsWI3Oogeh6w=
 github.com/google/flatbuffers v1.12.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8=
+github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
 github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM=
 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
@@ -62,17 +70,48 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.opencensus.io v0.22.5 h1:dntmOdLpSpHlVqbW5Eay97DelsZHe+55D+xC6i0dDS0=
+go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200918174421-af09f7315aff h1:1CPUrky56AcgSpxz/KfgzQWzfG09u5YOL8MvPYBlrL8=
 golang.org/x/sys v0.0.0-20200918174421-af09f7315aff/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb h1:i1Ppqkc3WQXikh8bXiwHqAN5Rv3/qDCcRk0/Otx73BY=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
diff --git a/levels.go b/levels.go
index 77fc50d..bf61839 100644
--- a/levels.go
+++ b/levels.go
@@ -18,6 +18,7 @@ package badger
 
 import (
 	"bytes"
+	"context"
 	"encoding/hex"
 	"fmt"
 	"math/rand"
@@ -28,7 +29,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	"golang.org/x/net/trace"
+	otrace "go.opencensus.io/trace"
 
 	"github.com/dgraph-io/badger/v2/pb"
 	"github.com/dgraph-io/badger/v2/table"
@@ -329,11 +330,14 @@ func (s *levelsController) dropPrefixes(prefixes [][]byte) error {
 		if len(tableGroups) == 0 {
 			continue
 		}
-
+		_, span := otrace.StartSpan(context.Background(), "Badger.Compaction")
+		span.Annotatef(nil, "Compaction level: %v", l.level)
+		span.Annotatef(nil, "Drop Prefixes: %v", prefixes)
+		defer span.End()
 		opt.Infof("Dropping prefix at level %d (%d tableGroups)", l.level, len(tableGroups))
 		for _, operation := range tableGroups {
 			cd := compactDef{
-				elog:         trace.New(fmt.Sprintf("Badger.L%d", l.level), "Compact"),
+				span:         span,
 				thisLevel:    l,
 				nextLevel:    l,
 				top:          nil,
@@ -485,6 +489,9 @@ func (s *levelsController) compactBuildTables(
 	y.NumCompactionTables.Add(numTables)
 	defer y.NumCompactionTables.Add(-numTables)
 
+	cd.span.Annotatef(nil, "Top tables count: %v Bottom tables count: %v",
+		len(topTables), len(botTables))
+
 	// Check overlap of the top level with the levels which are not being
 	// compacted in this compaction.
 	hasOverlap := s.checkOverlap(cd.allTables(), cd.nextLevel.level+1)
@@ -768,7 +775,7 @@ func containsAnyPrefixes(smallValue, largeValue []byte, listOfPrefixes [][]byte)
 }
 
 type compactDef struct {
-	elog trace.Trace
+	span *otrace.Span
 
 	thisLevel *levelHandler
 	nextLevel *levelHandler
@@ -961,15 +968,15 @@ var errFillTables = errors.New("Unable to fill tables")
 func (s *levelsController) doCompact(id int, p compactionPriority) error {
 	l := p.level
 	y.AssertTrue(l+1 < s.kv.opt.MaxLevels) // Sanity check.
-
+	_, span := otrace.StartSpan(context.Background(), "Badger.Compaction")
+	span.Annotatef(nil, "Compaction level: %v", l)
+	defer span.End()
 	cd := compactDef{
-		elog:         trace.New(fmt.Sprintf("Badger.L%d", l), "Compact"),
+		span:         span,
 		thisLevel:    s.levels[l],
 		nextLevel:    s.levels[l+1],
 		dropPrefixes: p.dropPrefixes,
 	}
-	cd.elog.SetMaxEvents(100)
-	defer cd.elog.Finish()
 
 	s.kv.opt.Debugf("[Compactor: %d] Attempting to run compaction: %+v", id, p)
 
diff --git a/value.go b/value.go
index ddedc11..862a83a 100644
--- a/value.go
+++ b/value.go
@@ -18,6 +18,7 @@ package badger
 
 import (
 	"bytes"
+	"context"
 	"fmt"
 	"hash"
 	"hash/crc32"
@@ -34,6 +35,7 @@ import (
 	"github.com/dgraph-io/badger/v2/y"
 	"github.com/dgraph-io/ristretto/z"
 	"github.com/pkg/errors"
+	otrace "go.opencensus.io/trace"
 )
 
 // maxVlogFileSize is the maximum size of the vlog file which can be created. Vlog Offset is of
@@ -853,7 +855,6 @@ func (vlog *valueLog) write(reqs []*request) error {
 			if err != nil {
 				return err
 			}
-			// TODO(Naman): Add a test to check that meta bits are correctly set?
 			// Restore the meta.
 			e.meta = tmpMeta
 
@@ -973,13 +974,13 @@ func (vlog *valueLog) pickLog(discardRatio float64) *logFile {
 	defer vlog.filesLock.RUnlock()
 
 LOOP:
-	// TODO(naman): Add a test for MaxDiscard which checks for the zero value.
 	// Pick a candidate that contains the largest amount of discardable data
 	fid, discard := vlog.discardStats.MaxDiscard()
 
 	// MaxDiscard will return fid=0 if it doesn't have any discard data. The
 	// vlog files start from 1.
 	if fid == 0 {
+		vlog.opt.Debugf("No file with discard stats")
 		return nil
 	}
 	lf, ok := vlog.filesMap[fid]
@@ -1038,6 +1039,9 @@ type reason struct {
 }
 
 func (vlog *valueLog) doRunGC(lf *logFile) error {
+	_, span := otrace.StartSpan(context.Background(), "Badger.GC")
+	span.Annotatef(nil, "GC rewrite for: %v", lf.path)
+	defer span.End()
 	if err := vlog.rewrite(lf); err != nil {
 		return err
 	}


Test patch:
diff --git a/db2_test.go b/db2_test.go
index afecd1b..ebf7b37 100644
--- a/db2_test.go
+++ b/db2_test.go
@@ -33,6 +33,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/dgraph-io/badger/v2/options"
+
 	"github.com/dgraph-io/badger/v2/pb"
 	"github.com/dgraph-io/badger/v2/table"
 	"github.com/dgraph-io/badger/v2/y"
@@ -904,3 +906,107 @@ func TestTxnReadTs(t *testing.T) {
 	require.NoError(t, err)
 	require.Equal(t, 1, int(db.orc.readTs()))
 }
+
+// This tests failed for stream writer with jemalloc and compression enabled.
+func TestKeyCount(t *testing.T) {
+	if !*manual {
+		t.Skip("Skipping test meant to be run manually.")
+		return
+	}
+
+	writeSorted := func(db *DB, num uint64) {
+		valSz := 128
+		value := make([]byte, valSz)
+		y.Check2(rand.Read(value))
+		es := 8 + valSz // key size is 8 bytes and value size is valSz
+
+		writer := db.NewStreamWriter()
+		require.NoError(t, writer.Prepare())
+
+		wg := &sync.WaitGroup{}
+		writeCh := make(chan *pb.KVList, 3)
+		writeRange := func(start, end uint64, streamId uint32) {
+			// end is not included.
+			defer wg.Done()
+			kvs := &pb.KVList{}
+			var sz int
+			for i := start; i < end; i++ {
+				key := make([]byte, 8)
+				binary.BigEndian.PutUint64(key, i)
+				kvs.Kv = append(kvs.Kv, &pb.KV{
+					Key:      key,
+					Value:    value,
+					Version:  1,
+					StreamId: streamId,
+				})
+
+				sz += es
+
+				if sz >= 4<<20 { // 4 MB
+					writeCh <- kvs
+					kvs = &pb.KVList{}
+					sz = 0
+				}
+			}
+			writeCh <- kvs
+		}
+
+		// Let's create some streams.
+		width := num / 16
+		streamID := uint32(0)
+		for start := uint64(0); start < num; start += width {
+			end := start + width
+			if end > num {
+				end = num
+			}
+			streamID++
+			wg.Add(1)
+			go writeRange(start, end, streamID)
+		}
+		go func() {
+			wg.Wait()
+			close(writeCh)
+		}()
+		for kvs := range writeCh {
+			require.NoError(t, writer.Write(kvs))
+		}
+		require.NoError(t, writer.Flush())
+	}
+
+	N := uint64(10 * 1e6) // 10 million entries
+	dir, err := ioutil.TempDir("", "badger-test")
+	require.NoError(t, err)
+	defer removeDir(dir)
+	opt := DefaultOptions(dir).
+		WithBlockCacheSize(100 << 20).
+		WithCompression(options.ZSTD)
+
+	db, err := Open(opt)
+	y.Check(err)
+	defer db.Close()
+	writeSorted(db, N)
+	require.NoError(t, db.Close())
+
+	// Read the db
+	db2, err := Open(DefaultOptions(dir))
+	y.Check(err)
+	defer db.Close()
+	lastKey := -1
+	count := 0
+	db2.View(func(txn *Txn) error {
+		iopt := DefaultIteratorOptions
+		iopt.AllVersions = true
+		it := txn.NewIterator(iopt)
+		defer it.Close()
+		for it.Rewind(); it.Valid(); it.Next() {
+			count++
+			i := it.Item()
+			key := binary.BigEndian.Uint64(i.Key())
+			// The following should happen as we're writing sorted data.
+			require.Equalf(t, lastKey+1, int(key), "Expected key: %d, Found Key: %d", lastKey+1, int(key))
+			lastKey = int(key)
+		}
+		return nil
+	})
+	require.Equal(t, N, uint64(count))
+}
diff --git a/discard_test.go b/discard_test.go
index 351273f..28c70a2 100644
--- a/discard_test.go
+++ b/discard_test.go
@@ -31,6 +31,10 @@ func TestDiscardStats(t *testing.T) {
 	opt := DefaultOptions(dir)
 	ds, err := initDiscardStats(opt)
 	require.NoError(t, err)
+	require.Zero(t, ds.nextEmptySlot)
+	fid, _ := ds.MaxDiscard()
+	require.Zero(t, fid)
+
 	for i := uint32(0); i < 20; i++ {
 		require.Equal(t, int64(i*100), ds.Update(i, int64(i*100)))
 	}
@@ -48,3 +52,26 @@ func TestDiscardStats(t *testing.T) {
 		require.Equal(t, int(id*100), int(val))
 	})
 }
+
+func TestReloadDiscardStats(t *testing.T) {
+	dir, err := ioutil.TempDir("", "badger-test")
+	require.NoError(t, err)
+	defer removeDir(dir)
+
+	opt := DefaultOptions(dir)
+	db, err := Open(opt)
+	require.NoError(t, err)
+	ds := db.vlog.discardStats
+
+	ds.Update(uint32(1), 1)
+	ds.Update(uint32(2), 1)
+	ds.Update(uint32(1), -1)
+	require.NoError(t, db.Close())
+
+	// Reopen the DB, discard stats should be same.
+	db2, err := Open(opt)
+	require.NoError(t, err)
+	ds2 := db2.vlog.discardStats
+	require.Zero(t, ds2.Update(uint32(1), 0))
+	require.Equal(t, 1, int(ds2.Update(uint32(2), 0)))
+}
diff --git a/manifest_test.go b/manifest_test.go
index e79bc2e..b2b5f0a 100644
--- a/manifest_test.go
+++ b/manifest_test.go
@@ -17,6 +17,7 @@
 package badger
 
 import (
+	"context"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
@@ -25,7 +26,7 @@ import (
 	"sort"
 	"testing"
 
-	"golang.org/x/net/trace"
+	otrace "go.opencensus.io/trace"
 
 	"github.com/dgraph-io/badger/v2/options"
 	"github.com/dgraph-io/badger/v2/pb"
@@ -171,11 +172,12 @@ func TestOverlappingKeyRangeError(t *testing.T) {
 
 	done := lh0.tryAddLevel0Table(t1)
 	require.Equal(t, true, done)
-
+	_, span := otrace.StartSpan(context.Background(), "Badger.Compaction")
+	span.Annotatef(nil, "Compaction level: %v", lh0)
 	cd := compactDef{
 		thisLevel: lh0,
 		nextLevel: lh1,
-		elog:      trace.New("Badger", "Compact"),
+		span:      span,
 	}
 
 	manifest := createManifest()
@@ -184,7 +186,10 @@ func TestOverlappingKeyRangeError(t *testing.T) {
 	done = lc.fillTablesL0(&cd)
 	require.Equal(t, true, done)
 	lc.runCompactDef(0, cd)
+	span.End()
 
+	_, span = otrace.StartSpan(context.Background(), "Badger.Compaction")
+	span.Annotatef(nil, "Compaction level: %v", lh0)
 	t2 := buildTestTable(t, "l", 2, opts)
 	defer t2.DecrRef()
 	done = lh0.tryAddLevel0Table(t2)
@@ -193,7 +198,7 @@ func TestOverlappingKeyRangeError(t *testing.T) {
 	cd = compactDef{
 		thisLevel: lh0,
 		nextLevel: lh1,
-		elog:      trace.New("Badger", "Compact"),
+		span:      span,
 	}
 	lc.fillTablesL0(&cd)
 	lc.runCompactDef(0, cd)
diff --git a/test.sh b/test.sh
index fdf899e..f7ea560 100755
--- a/test.sh
+++ b/test.sh
@@ -10,6 +10,29 @@ if [[ ! -z "$TEAMCITY_VERSION" ]]; then
   export GOFLAGS="-json"
 fi
 
+function InstallJemalloc() {
+  pushd .
+  if [ ! -f /usr/local/lib/libjemalloc.a ]; then
+		USER_ID=`id -u`
+    	JEMALLOC_URL="https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2"
+
+		mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ;
+		echo "Downloading jemalloc" ;
+		curl -s -L ${JEMALLOC_URL} -o jemalloc.tar.bz2 ;
+		tar xjf ./jemalloc.tar.bz2 ;
+		cd jemalloc-5.2.1 ;
+		./configure --with-jemalloc-prefix='je_' ;
+		make ;
+		if [ "$USER_ID" -eq "0" ]; then
+			make install ;
+		else
+			echo "==== Need sudo access to install jemalloc" ;
+			sudo make install ;
+		fi
+	fi
+  popd
+}
+
 # Ensure that we can compile the binary.
 pushd badger
 go build -v .
@@ -30,6 +53,10 @@ go test -v $tags -run='TestTruncateVlogNoClose2$' --manual=true
 go test -v $tags -run='TestTruncateVlogNoClose3$' --manual=true
 rm -rf p
 
+InstallJemalloc
+# Run the key count test for stream writer.
+go test -v -tags jemalloc -run='TestKeyCount' --manual=true
+
 # Run the normal tests.
 echo "==> Starting tests.. "
 # go test -timeout=25m -v -race github.com/dgraph-io/badger/v2/...
diff --git a/value_test.go b/value_test.go
index c351da4..6ba065d 100644
--- a/value_test.go
+++ b/value_test.go
@@ -1150,3 +1150,60 @@ func TestValidateWrite(t *testing.T) {
 	err = log.validateWrites([]*request{req1, req})
 	require.Error(t, err)
 }
+
+func TestValueLogMeta(t *testing.T) {
+	dir, err := ioutil.TempDir("", "badger-test")
+	y.Check(err)
+	defer removeDir(dir)
+
+	opt := getTestOptions(dir).WithValueThreshold(16)
+	db, _ := Open(opt)
+	defer db.Close()
+	txn := db.NewTransaction(true)
+	for i := 0; i < 10; i++ {
+		k := []byte(fmt.Sprintf("key=%d", i))
+		v := []byte(fmt.Sprintf("val=%020d", i))
+		require.NoError(t, txn.SetEntry(NewEntry(k, v)))
+	}
+	require.NoError(t, txn.Commit())
+	fids := db.vlog.sortedFids()
+	require.Equal(t, 1, len(fids))
+
+	// vlog entries must not have txn meta.
+	db.vlog.filesMap[fids[0]].iterate(true, 0, func(e Entry, vp valuePointer) error {
+		require.Zero(t, e.meta&(bitTxn|bitFinTxn))
+		return nil
+	})
+
+	// Entries in LSM tree must have txn bit of meta set
+	txn = db.NewTransaction(false)
+	defer txn.Discard()
+	iopt := DefaultIteratorOptions
+	key := []byte("key")
+	iopt.Prefix = key
+	itr := txn.NewIterator(iopt)
+	defer itr.Close()
+	var count int
+	for itr.Seek(key); itr.ValidForPrefix(key); itr.Next() {
+		item := itr.Item()
+		require.Equal(t, bitTxn, item.meta&(bitTxn|bitFinTxn))
+		count++
+	}
+	require.Equal(t, 10, count)
+}
+
+// This tests asserts the condition that vlog fids start from 1.
+// TODO(naman): should this be changed to assert instead?
+func TestFirstVlogFile(t *testing.T) {
+	dir, err := ioutil.TempDir("", "badger-test")
+	require.NoError(t, err)
+	defer removeDir(dir)
+
+	opt := DefaultOptions(dir)
+	db, err := Open(opt)
+	defer db.Close()
+
+	fids := db.vlog.sortedFids()
+	require.NotZero(t, len(fids))
+	require.Equal(t, uint32(1), fids[0])
+}