Repository: dgraph-io/badger
Pull request #1476 (2020-08-25 15:33:07)

release/v2.2007 - Rework Block and Index cache (#1473)
There were a bunch of conflicts. Please review it thoroughly.

This PR separates the block cache and the index cache. Earlier we had a mix of both the caches.
Block Cache -> Stores block. Should be used when running badger with compression/encryption
Index Cache -> Used to limit the memory used by table indices and bloom filters.

This PR also remove the `KeepBlocksInCache` and `KeepBlockIndicesInCache` options.

(cherry picked from commit c3b15824dc149149616d474e5e150fa8ca337d7f)

<!-- Reviewable:start -->
---
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/dgraph-io/badger/1476)
<!-- Reviewable:end -->



Patch:
diff --git a/db.go b/db.go
index b6edb72..6818661 100644
--- a/db.go
+++ b/db.go
@@ -90,7 +90,7 @@ type DB struct {
 	pub        *publisher
 	registry   *KeyRegistry
 	blockCache *ristretto.Cache
-	bfCache    *ristretto.Cache
+	indexCache *ristretto.Cache
 }
 
 const (
@@ -311,11 +311,11 @@ func Open(opt Options) (db *DB, err error) {
 		}
 	}()
 
-	if opt.MaxCacheSize > 0 {
+	if opt.BlockCacheSize > 0 {
 		config := ristretto.Config{
 			// Use 5% of cache memory for storing counters.
-			NumCounters: int64(float64(opt.MaxCacheSize) * 0.05 * 2),
-			MaxCost:     int64(float64(opt.MaxCacheSize) * 0.95),
+			NumCounters: int64(float64(opt.BlockCacheSize) * 0.05 * 2),
+			MaxCost:     int64(float64(opt.BlockCacheSize) * 0.95),
 			BufferItems: 64,
 			Metrics:     true,
 		}
@@ -325,15 +325,15 @@ func Open(opt Options) (db *DB, err error) {
 		}
 	}
 
-	if opt.MaxBfCacheSize > 0 {
+	if opt.IndexCacheSize > 0 {
 		config := ristretto.Config{
 			// Use 5% of cache memory for storing counters.
-			NumCounters: int64(float64(opt.MaxBfCacheSize) * 0.05 * 2),
-			MaxCost:     int64(float64(opt.MaxBfCacheSize) * 0.95),
+			NumCounters: int64(float64(opt.IndexCacheSize) * 0.05 * 2),
+			MaxCost:     int64(float64(opt.IndexCacheSize) * 0.95),
 			BufferItems: 64,
 			Metrics:     true,
 		}
-		db.bfCache, err = ristretto.NewCache(&config)
+		db.indexCache, err = ristretto.NewCache(&config)
 		if err != nil {
 			return nil, errors.Wrap(err, "failed to create bf cache")
 		}
@@ -427,7 +427,7 @@ func Open(opt Options) (db *DB, err error) {
 // cleanup goroutines in case of an error.
 func (db *DB) cleanup() {
 	db.blockCache.Close()
-	db.bfCache.Close()
+	db.indexCache.Close()
 	db.stopMemoryFlush()
 	db.stopCompactions()
 
@@ -448,18 +448,18 @@ func (db *DB) cleanup() {
 	db.vlog.Close()
 }
 
-// DataCacheMetrics returns the metrics for the underlying data cache.
-func (db *DB) DataCacheMetrics() *ristretto.Metrics {
+// BlockCacheMetrics returns the metrics for the underlying block cache.
+func (db *DB) BlockCacheMetrics() *ristretto.Metrics {
 	if db.blockCache != nil {
 		return db.blockCache.Metrics
 	}
 	return nil
 }
 
-// BfCacheMetrics returns the metrics for the underlying bloom filter cache.
-func (db *DB) BfCacheMetrics() *ristretto.Metrics {
-	if db.bfCache != nil {
-		return db.bfCache.Metrics
+// IndexCacheMetrics returns the metrics for the underlying index cache.
+func (db *DB) IndexCacheMetrics() *ristretto.Metrics {
+	if db.indexCache != nil {
+		return db.indexCache.Metrics
 	}
 	return nil
 }
@@ -553,7 +553,7 @@ func (db *DB) close() (err error) {
 	db.closers.updateSize.SignalAndWait()
 	db.orc.Stop()
 	db.blockCache.Close()
-	db.bfCache.Close()
+	db.indexCache.Close()
 
 	if db.opt.InMemory {
 		return
@@ -1042,8 +1042,8 @@ func (db *DB) handleFlushTask(ft flushTask) error {
 	bopts := buildTableOptions(db.opt)
 	bopts.DataKey = dk
 	// Builder does not need cache but the same options are used for opening table.
-	bopts.Cache = db.blockCache
-	bopts.BfCache = db.bfCache
+	bopts.BlockCache = db.blockCache
+	bopts.IndexCache = db.indexCache
 	tableData := buildL0Table(ft, bopts)
 
 	fileID := db.lc.reserveFileID()
@@ -1626,7 +1626,7 @@ func (db *DB) dropAll() (func(), error) {
 	db.lc.nextFileID = 1
 	db.opt.Infof("Deleted %d value log files. DropAll done.\n", num)
 	db.blockCache.Clear()
-	db.bfCache.Clear()
+	db.indexCache.Clear()
 
 	return resume, nil
 }
diff --git a/levels.go b/levels.go
index ccef697..aaef7e3 100644
--- a/levels.go
+++ b/levels.go
@@ -157,8 +157,8 @@ func newLevelsController(db *DB, mf *Manifest) (*levelsController, error) {
 			// Set compression from table manifest.
 			topt.Compression = tf.Compression
 			topt.DataKey = dk
-			topt.Cache = db.blockCache
-			topt.BfCache = db.bfCache
+			topt.BlockCache = db.blockCache
+			topt.IndexCache = db.indexCache
 			t, err := table.OpenTable(fd, topt)
 			if err != nil {
 				if strings.HasPrefix(err.Error(), "CHECKSUM_MISMATCH:") {
@@ -569,8 +569,8 @@ nextTable:
 		bopts := buildTableOptions(s.kv.opt)
 		bopts.DataKey = dk
 		// Builder does not need cache but the same options are used for opening table.
-		bopts.Cache = s.kv.blockCache
-		bopts.BfCache = s.kv.bfCache
+		bopts.BlockCache = s.kv.blockCache
+		bopts.IndexCache = s.kv.indexCache
 		builder := table.NewTableBuilder(bopts)
 		var numKeys, numSkips uint64
 		for ; it.Valid(); it.Next() {
diff --git a/options.go b/options.go
index 4983920..5cefd88 100644
--- a/options.go
+++ b/options.go
@@ -62,8 +62,8 @@ type Options struct {
 	BlockSize          int
 	BloomFalsePositive float64
 	KeepL0InMemory     bool
-	MaxCacheSize       int64
-	MaxBfCacheSize     int64
+	BlockCacheSize     int64
+	IndexCacheSize     int64
 	LoadBloomsOnOpen   bool
 
 	NumLevelZeroTables      int
@@ -98,12 +98,6 @@ type Options struct {
 	// conflict detection is disabled.
 	DetectConflicts bool
 
-	// KeepBlockIndicesInCache decides whether to keep the block offsets in the cache or not.
-	KeepBlockIndicesInCache bool
-
-	// KeepBlocksInCache decides whether to keep the sst blocks in the cache or not.
-	KeepBlocksInCache bool
-
 	// Transaction start and commit timestamps are managed by end-user.
 	// This is only useful for databases built on top of Badger (like Dgraph).
 	// Not recommended for most users.
@@ -141,9 +135,10 @@ func DefaultOptions(path string) Options {
 		KeepL0InMemory:          false,
 		VerifyValueChecksum:     false,
 		Compression:             options.None,
-		MaxCacheSize:            0,
-		MaxBfCacheSize:          0,
+		BlockCacheSize:          0,
+		IndexCacheSize:          0,
 		LoadBloomsOnOpen:        true,
+
 		// The following benchmarks were done on a 4 KB block size (default block size). The
 		// compression is ratio supposed to increase with increasing compression level but since the
 		// input for compression algorithm is small (4 KB), we don't get significant benefit at
@@ -169,22 +164,18 @@ func DefaultOptions(path string) Options {
 		EncryptionKey:                 []byte{},
 		EncryptionKeyRotationDuration: 10 * 24 * time.Hour, // Default 10 days.
 		DetectConflicts:               true,
-		KeepBlocksInCache:             false,
-		KeepBlockIndicesInCache:       false,
 	}
 }
 
 func buildTableOptions(opt Options) table.Options {
 	return table.Options{
-		BlockSize:               opt.BlockSize,
-		BloomFalsePositive:      opt.BloomFalsePositive,
-		LoadBloomsOnOpen:        opt.LoadBloomsOnOpen,
-		LoadingMode:             opt.TableLoadingMode,
-		ChkMode:                 opt.ChecksumVerificationMode,
-		Compression:             opt.Compression,
-		ZSTDCompressionLevel:    opt.ZSTDCompressionLevel,
-		KeepBlockIndicesInCache: opt.KeepBlockIndicesInCache,
-		KeepBlocksInCache:       opt.KeepBlocksInCache,
+		BlockSize:            opt.BlockSize,
+		BloomFalsePositive:   opt.BloomFalsePositive,
+		LoadBloomsOnOpen:     opt.LoadBloomsOnOpen,
+		LoadingMode:          opt.TableLoadingMode,
+		ChkMode:              opt.ChecksumVerificationMode,
+		Compression:          opt.Compression,
+		ZSTDCompressionLevel: opt.ZSTDCompressionLevel,
 	}
 }
 
@@ -500,7 +491,7 @@ func (opt Options) WithEncryptionKey(key []byte) Options {
 	return opt
 }
 
-// WithEncryptionRotationDuration returns new Options value with the duration set to
+// WithEncryptionKeyRotationDuration returns new Options value with the duration set to
 // the given value.
 //
 // Key Registry will use this duration to create new keys. If the previous generated
@@ -559,18 +550,18 @@ func (opt Options) WithChecksumVerificationMode(cvMode options.ChecksumVerificat
 	return opt
 }
 
-// WithMaxCacheSize returns a new Options value with MaxCacheSize set to the given value.
+// WithBlockCacheSize returns a new Options value with BlockCacheSize set to the given value.
 //
-// This value specifies how much data cache should hold in memory. A small size of cache means lower
-// memory consumption and lookups/iterations would take longer.
-// It is recommended to use a cache if you're using compression or encryption.
+// This value specifies how much data cache should hold in memory. A small size
+// of cache means lower memory consumption and lookups/iterations would take
+// longer. It is recommended to use a cache if you're using compression or encryption.
 // If compression and encryption both are disabled, adding a cache will lead to
-// unnecessary overhead which will affect the read performance. Setting size to zero disables the
-// cache altogether.
+// unnecessary overhead which will affect the read performance. Setting size to
+// zero disables the cache altogether.
 //
-// Default value of MaxCacheSize is zero.
-func (opt Options) WithMaxCacheSize(size int64) Options {
-	opt.MaxCacheSize = size
+// Default value of BlockCacheSize is zero.
+func (opt Options) WithBlockCacheSize(size int64) Options {
+	opt.BlockCacheSize = size
 	return opt
 }
 
@@ -619,22 +610,6 @@ func (opt Options) WithBypassLockGuard(b bool) Options {
 	return opt
 }
 
-// WithMaxBfCacheSize returns a new Options value with MaxBfCacheSize set to the given value.
-//
-// This value specifies how much memory should be used by the bloom filters.
-// Badger uses bloom filters to speed up lookups. Each table has its own bloom
-// filter and each bloom filter is approximately of 5 MB.
-//
-// Zero value for BfCacheSize means all the bloom filters will be kept in
-// memory and the cache is disabled.
-//
-// The default value of MaxBfCacheSize is 0 which means all bloom filters will
-// be kept in memory.
-func (opt Options) WithMaxBfCacheSize(size int64) Options {
-	opt.MaxBfCacheSize = size
-	return opt
-}
-
 // WithLoadBloomsOnOpen returns a new Options value with LoadBloomsOnOpen set to the given value.
 //
 // Badger uses bloom filters to speed up key lookups. When LoadBloomsOnOpen is set
@@ -648,6 +623,24 @@ func (opt Options) WithLoadBloomsOnOpen(b bool) Options {
 	return opt
 }
 
+// WithIndexCacheSize returns a new Options value with IndexCacheSize set to
+// the given value.
+//
+// This value specifies how much memory should be used by table indices. These
+// indices include the block offsets and the bloomfilters. Badger uses bloom
+// filters to speed up lookups. Each table has its own bloom
+// filter and each bloom filter is approximately of 5 MB.
+//
+// Zero value for IndexCacheSize means all the indices will be kept in
+// memory and the cache is disabled.
+//
+// The default value of IndexCacheSize is 0 which means all indices are kept in
+// memory.
+func (opt Options) WithIndexCacheSize(size int64) Options {
+	opt.IndexCacheSize = size
+	return opt
+}
+
 // WithDetectConflicts returns a new Options value with DetectConflicts set to the given value.
 //
 // Detect conflicts options determines if the transactions would be checked for
@@ -661,38 +654,3 @@ func (opt Options) WithDetectConflicts(b bool) Options {
 	opt.DetectConflicts = b
 	return opt
 }
-
-// WithKeepBlockIndicesInCache returns a new Option value with KeepBlockOffsetInCache set to the
-// given value.
-//
-// When this option is set badger will store the block offsets in a cache along with the blocks.
-// The size of the cache is determined by the MaxCacheSize option.If the MaxCacheSize is set to
-// zero, then MaxCacheSize is set to 100 mb. When indices are stored in the cache, the read
-// performance might be affected but the cache limits the amount of memory used by the indices.
-//
-// The default value of KeepBlockOffsetInCache is false.
-func (opt Options) WithKeepBlockIndicesInCache(val bool) Options {
-	opt.KeepBlockIndicesInCache = val
-
-	if val && opt.MaxCacheSize == 0 {
-		opt.MaxCacheSize = 100 << 20
-	}
-	return opt
-}
-
-// WithKeepBlocksInCache returns a new Option value with KeepBlocksInCache set to the
-// given value.
-//
-// When this option is set badger will store the block in the cache. The size of the cache is
-// determined by the MaxCacheSize option.If the MaxCacheSize is set to zero,
-// then MaxCacheSize is set to 100 mb.
-//
-// The default value of KeepBlocksInCache is false.
-func (opt Options) WithKeepBlocksInCache(val bool) Options {
-	opt.KeepBlocksInCache = val
-
-	if val && opt.MaxCacheSize == 0 {
-		opt.MaxCacheSize = 100 << 20
-	}
-	return opt
-}
diff --git a/stream_writer.go b/stream_writer.go
index 3ed865c..38ffba2 100644
--- a/stream_writer.go
+++ b/stream_writer.go
@@ -417,8 +417,8 @@ func (w *sortedWriter) createTable(builder *table.Builder) error {
 	fileID := w.db.lc.reserveFileID()
 	opts := buildTableOptions(w.db.opt)
 	opts.DataKey = builder.DataKey()
-	opts.Cache = w.db.blockCache
-	opts.BfCache = w.db.bfCache
+	opts.BlockCache = w.db.blockCache
+	opts.IndexCache = w.db.indexCache
 	var tbl *table.Table
 	if w.db.opt.InMemory {
 		var err error
diff --git a/table/table.go b/table/table.go
index a9b3e45..90e8df3 100644
--- a/table/table.go
+++ b/table/table.go
@@ -76,8 +76,8 @@ type Options struct {
 	// Compression indicates the compression algorithm used for block compression.
 	Compression options.CompressionType
 
-	Cache   *ristretto.Cache
-	BfCache *ristretto.Cache
+	BlockCache *ristretto.Cache
+	IndexCache *ristretto.Cache
 
 	// ZSTDCompressionLevel is the ZSTD compression level used for compressing blocks.
 	ZSTDCompressionLevel int
@@ -85,12 +85,6 @@ type Options struct {
 	// When LoadBloomsOnOpen is set, bloom filters will be read only when they are accessed.
 	// Otherwise they will be loaded on table open.
 	LoadBloomsOnOpen bool
-
-	// KeepBlockIndicesInCache decides whether to keep the block offsets in the cache or not.
-	KeepBlockIndicesInCache bool
-
-	// KeepBlocksInCache decides whether to keep the block in the cache or not.
-	KeepBlocksInCache bool
 }
 
 // TableInterface is useful for testing.
@@ -108,9 +102,9 @@ type Table struct {
 	tableSize int      // Initialized in OpenTable, using fd.Stat().
 	bfLock    sync.Mutex
 
-	blockIndex []*pb.BlockOffset
-	ref        int32    // For file garbage collection. Atomic.
-	bf         *z.Bloom // Nil if BfCache is set.
+	blockOffset []*pb.BlockOffset
+	ref         int32    // For file garbage collection. Atomic.
+	bf          *z.Bloom // Nil if index cache in enabled.
 
 	mmap []byte // Memory mapped.
 
@@ -172,11 +166,11 @@ func (t *Table) DecrRef() error {
 		}
 		// Delete all blocks from the cache.
 		for i := 0; i < t.noOfBlocks; i++ {
-			t.opt.Cache.Del(t.blockCacheKey(i))
+			t.opt.BlockCache.Del(t.blockCacheKey(i))
 		}
-		// Delete bloom filter from the cache.
-		t.opt.BfCache.Del(t.bfCacheKey())
-
+		// Delete bloom filter and indices from the cache.
+		t.opt.IndexCache.Del(t.blockOffsetsCacheKey())
+		t.opt.IndexCache.Del(t.bfCacheKey())
 	}
 	return nil
 }
@@ -296,7 +290,7 @@ func OpenInMemoryTable(data []byte, id uint64, opt *Options) (*Table, error) {
 func (t *Table) initBiggestAndSmallest() error {
 	var err error
 	var ko *pb.BlockOffset
-	if ko, err = t.readIndex(); err != nil {
+	if ko, err = t.initIndex(); err != nil {
 		return errors.Wrapf(err, "failed to read index.")
 	}
 
@@ -347,9 +341,9 @@ func (t *Table) readNoFail(off, sz int) []byte {
 	return res
 }
 
-// readIndex reads the index and populate the necessary table fields and returns
+// initIndex reads the index and populate the necessary table fields and returns
 // first block offset
-func (t *Table) readIndex() (*pb.BlockOffset, error) {
+func (t *Table) initIndex() (*pb.BlockOffset, error) {
 	readPos := t.tableSize
 
 	// Read checksum len from the last 4 bytes.
@@ -382,54 +376,54 @@ func (t *Table) readIndex() (*pb.BlockOffset, error) {
 		return nil, y.Wrapf(err, "failed to verify checksum for table: %s", t.Filename())
 	}
 
-	index := pb.TableIndex{}
-	// Decrypt the table index if it is encrypted.
-	if t.shouldDecrypt() {
-		var err error
-		if data, err = t.decrypt(data); err != nil {
-			return nil, y.Wrapf(err,
-				"Error while decrypting table index for the table %d in Table.readIndex", t.id)
-		}
+	index, err := t.readTableIndex()
+	if err != nil {
+		return nil, err
 	}
-	err := proto.Unmarshal(data, &index)
-	y.Check(err)
 
 	t.estimatedSize = index.EstimatedSize
 	t.noOfBlocks = len(index.Offsets)
 
-	if t.opt.LoadBloomsOnOpen {
-		t.bfLock.Lock()
-		t.bf, _ = t.readBloomFilter()
-		t.bfLock.Unlock()
-	}
-
-	if t.opt.KeepBlockIndicesInCache && t.opt.Cache != nil {
-		t.opt.Cache.Set(
-			t.blockOffsetsCacheKey(),
-			index.Offsets,
-			calculateOffsetsSize(index.Offsets))
-
-		return index.Offsets[0], nil
+	// No cache
+	if t.opt.IndexCache == nil {
+		if t.opt.LoadBloomsOnOpen {
+			bf, err := z.JSONUnmarshal(index.BloomFilter)
+			if err != nil {
+				return nil,
+					errors.Wrapf(err, "failed to unmarshal bloomfilter for table:%d", t.id)
+			}
+			// Keep blooms in memory.
+			t.bfLock.Lock()
+			t.bf = bf
+			t.bfLock.Unlock()
+		}
+		// Keep block offsets in memory since there is no cache.
+		t.blockOffset = index.Offsets
 	}
 
-	t.blockIndex = index.Offsets
+	// We don't need to put anything in the indexCache here. Table.Open will
+	// create an iterator and that iterator will push the indices in cache.
 	return index.Offsets[0], nil
 }
 
 // blockOffsets returns block offsets of this table.
 func (t *Table) blockOffsets() []*pb.BlockOffset {
-	if !t.opt.KeepBlockIndicesInCache || t.opt.Cache == nil {
-		return t.blockIndex
+	if t.opt.IndexCache == nil {
+		return t.blockOffset
 	}
 
-	if val, ok := t.opt.Cache.Get(t.blockOffsetsCacheKey()); ok && val != nil {
+	if val, ok := t.opt.IndexCache.Get(t.blockOffsetsCacheKey()); ok && val != nil {
 		return val.([]*pb.BlockOffset)
 	}
 
-	ti := t.readTableIndex()
+	index, err := t.readTableIndex()
+	y.Check(err)
+	t.opt.IndexCache.Set(
+		t.blockOffsetsCacheKey(),
+		index.Offsets,
+		calculateOffsetsSize(index.Offsets))
 
-	t.opt.Cache.Set(t.blockOffsetsCacheKey(), ti.Offsets, calculateOffsetsSize(ti.Offsets))
-	return ti.Offsets
+	return index.Offsets
 }
 
 // calculateOffsetsSize returns the size of *pb.BlockOffset array
@@ -451,9 +445,9 @@ func (t *Table) block(idx int) (*block, error) {
 	if idx >= t.noOfBlocks {
 		return nil, errors.New("block out of index")
 	}
-	if t.opt.Cache != nil && t.opt.KeepBlocksInCache {
+	if t.opt.BlockCache != nil {
 		key := t.blockCacheKey(idx)
-		blk, ok := t.opt.Cache.Get(key)
+		blk, ok := t.opt.BlockCache.Get(key)
 		if ok && blk != nil {
 			return blk.(*block), nil
 		}
@@ -518,14 +512,14 @@ func (t *Table) block(idx int) (*block, error) {
 			return nil, err
 		}
 	}
-	if t.opt.Cache != nil && t.opt.KeepBlocksInCache {
+	if t.opt.BlockCache != nil {
 		key := t.blockCacheKey(idx)
-		t.opt.Cache.Set(key, blk, blk.size())
+		t.opt.BlockCache.Set(key, blk, blk.size())
 	}
 	return blk, nil
 }
 
-// bfCacheKey returns the cache key for bloom filter.
+// bfCacheKey returns the cache key for bloom filter. Bloom filters are stored in index cache.
 func (t *Table) bfCacheKey() []byte {
 	y.AssertTrue(t.id < math.MaxUint32)
 	buf := make([]byte, 6)
@@ -537,6 +531,7 @@ func (t *Table) bfCacheKey() []byte {
 	return buf
 }
 
+// blockCacheKey is used to store blocks in the block cache.
 func (t *Table) blockCacheKey(idx int) []byte {
 	y.AssertTrue(t.id < math.MaxUint32)
 	y.AssertTrue(uint32(idx) < math.MaxUint32)
@@ -548,15 +543,10 @@ func (t *Table) blockCacheKey(idx int) []byte {
 	return buf
 }
 
-// blockOffsetsCacheKey returns the cache key for block offsets.
-func (t *Table) blockOffsetsCacheKey() []byte {
-	y.AssertTrue(t.id < math.MaxUint32)
-	buf := make([]byte, 6)
-	buf[0] = 'b'
-	buf[1] = 'o'
-
-	binary.BigEndian.PutUint32(buf[2:], uint32(t.id))
-	return buf
+// blockOffsetsCacheKey returns the cache key for block offsets. blockOffsets
+// are stored in the index cache.
+func (t *Table) blockOffsetsCacheKey() uint64 {
+	return t.id
 }
 
 // EstimatedSize returns the total size of key-values stored in this table (including the
@@ -581,28 +571,25 @@ func (t *Table) ID() uint64 { return t.id }
 // DoesNotHave returns true if (but not "only if") the table does not have the key hash.
 // It does a bloom filter lookup.
 func (t *Table) DoesNotHave(hash uint64) bool {
-	var bf *z.Bloom
-
 	// Return fast if the cache is absent.
-	if t.opt.BfCache == nil {
+	if t.opt.IndexCache == nil {
 		t.bfLock.Lock()
-		// Load bloomfilter into memory if the cache is absent.
 		if t.bf == nil {
 			y.AssertTrue(!t.opt.LoadBloomsOnOpen)
+			// Load bloomfilter into memory since the cache is absent.
 			t.bf, _ = t.readBloomFilter()
 		}
 		t.bfLock.Unlock()
 		return !t.bf.Has(hash)
 	}
 
-	// Check if the bloomfilter exists in the cache.
-	if b, ok := t.opt.BfCache.Get(t.bfCacheKey()); b != nil && ok {
-		bf = b.(*z.Bloom)
-		return !bf.Has(hash)
+	// Check if the bloom filter exists in the cache.
+	if bf, ok := t.opt.IndexCache.Get(t.bfCacheKey()); bf != nil && ok {
+		return !bf.(*z.Bloom).Has(hash)
 	}
 
 	bf, sz := t.readBloomFilter()
-	t.opt.BfCache.Set(t.bfCacheKey(), bf, int64(sz))
+	t.opt.IndexCache.Set(t.bfCacheKey(), bf, int64(sz))
 	return !bf.Has(hash)
 }
 
@@ -610,24 +597,28 @@ func (t *Table) DoesNotHave(hash uint64) bool {
 // along with the bloom filter.
 func (t *Table) readBloomFilter() (*z.Bloom, int) {
 	// Read bloom filter from the SST.
-	index := t.readTableIndex()
+	index, err := t.readTableIndex()
+	y.Check(err)
+
 	bf, err := z.JSONUnmarshal(index.BloomFilter)
 	y.Check(err)
 	return bf, len(index.BloomFilter)
 }
 
 // readTableIndex reads table index from the sst and returns its pb format.
-func (t *Table) readTableIndex() *pb.TableIndex {
+func (t *Table) readTableIndex() (*pb.TableIndex, error) {
 	data := t.readNoFail(t.indexStart, t.indexLen)
 	index := pb.TableIndex{}
 	var err error
 	// Decrypt the table index if it is encrypted.
 	if t.shouldDecrypt() {
-		data, err = t.decrypt(data)
-		y.Check(err)
+		if data, err = t.decrypt(data); err != nil {
+			return nil, y.Wrapf(err,
+				"Error while decrypting table index for the table %d in readTableIndex", t.id)
+		}
 	}
 	y.Check(proto.Unmarshal(data, &index))
-	return &index
+	return &index, nil
 }
 
 // VerifyChecksum verifies checksum for all blocks of table. This function is called by


Test patch:
diff --git a/db_test.go b/db_test.go
index 78b4f29..4a2bd40 100644
--- a/db_test.go
+++ b/db_test.go
@@ -73,7 +73,7 @@ func getTestOptions(dir string) Options {
 		WithMaxTableSize(1 << 15). // Force more compaction.
 		WithLevelOneSize(4 << 15). // Force more compaction.
 		WithSyncWrites(false).
-		WithMaxCacheSize(10 << 20)
+		WithBlockCacheSize(10 << 20)
 	if !*mmap {
 		return opt.WithValueLogLoadingMode(options.FileIO)
 	}
@@ -287,12 +287,11 @@ func TestGet(t *testing.T) {
 		test(t, db)
 		require.NoError(t, db.Close())
 	})
-	t.Run("cache disabled", func(t *testing.T) {
-		opts := DefaultOptions("").WithInMemory(true).WithMaxCacheSize(0)
-		db, err := Open(opts)
-		require.NoError(t, err)
-		test(t, db)
-		require.NoError(t, db.Close())
+	t.Run("cache enabled", func(t *testing.T) {
+		opts := DefaultOptions("").WithBlockCacheSize(10 << 20)
+		runBadgerTest(t, &opts, func(t *testing.T, db *DB) {
+			test(t, db)
+		})
 	})
 }
 
diff --git a/iterator_test.go b/iterator_test.go
index 0fa45cd..cdd3276 100644
--- a/iterator_test.go
+++ b/iterator_test.go
@@ -210,7 +210,7 @@ func TestIteratePrefix(t *testing.T) {
 	t.Run("With Block Offsets in Cache", func(t *testing.T) {
 		t.Parallel()
 		opts := getTestOptions("")
-		opts = opts.WithKeepBlockIndicesInCache(true)
+		opts.IndexCacheSize = 100 << 20
 		runBadgerTest(t, &opts, func(t *testing.T, db *DB) {
 			testIteratorPrefix(t, db)
 		})
@@ -219,7 +219,8 @@ func TestIteratePrefix(t *testing.T) {
 	t.Run("With Block Offsets and Blocks in Cache", func(t *testing.T) {
 		t.Parallel()
 		opts := getTestOptions("")
-		opts = opts.WithKeepBlockIndicesInCache(true).WithKeepBlocksInCache(true)
+		opts.BlockCacheSize = 100 << 20
+		opts.IndexCacheSize = 100 << 20
 		runBadgerTest(t, &opts, func(t *testing.T, db *DB) {
 			testIteratorPrefix(t, db)
 		})
@@ -228,7 +229,7 @@ func TestIteratePrefix(t *testing.T) {
 	t.Run("With Blocks in Cache", func(t *testing.T) {
 		t.Parallel()
 		opts := getTestOptions("")
-		opts = opts.WithKeepBlocksInCache(true)
+		opts.BlockCacheSize = 100 << 20
 		runBadgerTest(t, &opts, func(t *testing.T, db *DB) {
 			testIteratorPrefix(t, db)
 		})
diff --git a/table/builder_test.go b/table/builder_test.go
index 34e0c28..5f22b39 100644
--- a/table/builder_test.go
+++ b/table/builder_test.go
@@ -38,7 +38,7 @@ func TestTableIndex(t *testing.T) {
 		f := buildTestTable(t, keyPrefix, 1, opts)
 		tbl, err := OpenTable(f, opts)
 		require.NoError(t, err)
-		require.Len(t, tbl.blockIndex, 1)
+		require.Len(t, tbl.blockOffset, 1)
 	})
 
 	t.Run("multiple keys", func(t *testing.T) {
@@ -88,7 +88,9 @@ func TestTableIndex(t *testing.T) {
 
 			// Ensure index is built correctly
 			require.Equal(t, blockCount, tbl.noOfBlocks)
-			for i, ko := range tbl.readTableIndex().Offsets {
+			idx, err := tbl.readTableIndex()
+			require.NoError(t, err)
+			for i, ko := range idx.Offsets {
 				require.Equal(t, ko.Key, blockFirstKeys[i])
 			}
 			f.Close()
diff --git a/table/table_test.go b/table/table_test.go
index 8d3ce0a..ab84427 100644
--- a/table/table_test.go
+++ b/table/table_test.go
@@ -788,7 +788,7 @@ func BenchmarkReadAndBuild(b *testing.B) {
 	for i := 0; i < b.N; i++ {
 		func() {
 			opts := Options{Compression: options.ZSTD, BlockSize: 4 * 0124, BloomFalsePositive: 0.01}
-			opts.Cache = cache
+			opts.BlockCache = cache
 			newBuilder := NewTableBuilder(opts)
 			it := tbl.NewIterator(false)
 			defer it.Close()
@@ -814,7 +814,7 @@ func BenchmarkReadMerged(b *testing.B) {
 	for i := 0; i < m; i++ {
 		filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63())
 		opts := Options{Compression: options.ZSTD, BlockSize: 4 * 1024, BloomFalsePositive: 0.01}
-		opts.Cache = cache
+		opts.BlockCache = cache
 		builder := NewTableBuilder(opts)
 		f, err := y.OpenSyncedFile(filename, true)
 		y.Check(err)
@@ -906,7 +906,7 @@ func getTableForBenchmarks(b *testing.B, count int, cache *ristretto.Cache) *Tab
 		cache, err = ristretto.NewCache(&cacheConfig)
 		require.NoError(b, err)
 	}
-	opts.Cache = cache
+	opts.BlockCache = cache
 	builder := NewTableBuilder(opts)
 	filename := fmt.Sprintf("%s%s%d.sst", os.TempDir(), string(os.PathSeparator), rand.Int63())
 	f, err := y.OpenSyncedFile(filename, true)