Repository: cockroachdb/pebble
Pull request #3608 (2024-05-11 02:22:40)

objstorage,sstable: add read-before for reader creation and iter index/filter blocks
We consider two use cases for read-before when using a
objstorageprovider.remoteReadable, given the high latency (and cost) of
each read operation:

- When a sstable.Reader is opened, it needs to read the footer, metaindex
  block and meta properties block. It starts by reading the footer which is
  at the end of the table and then proceeds to read the other two. Instead
  of doing 3 tiny reads, we would like to do one read.

- When a single-level or two-level iterator is opened, it reads the
  (top-level) index block first. When the iterator is used, it will
  typically follow this by reading the filter block (since SeeKPrefixGE is
  common in CockroachDB). For a two-level iterator it will also read the
  lower-level index blocks which are after the filter block and before the
  top-level index block. It would be ideal if reading the top-level index
  block read enough to include the filter block. And for two-level
  iterators this would also include the lower-level index blocks.

In both use-cases we want the first read from the remoteReadable to do a
larger read, and read bytes earlier than the requested read, hence
"read-before". Subsequent reads from the remoteReadable can use the usual
readahead logic (for the second use-case above, this can help with
sequential reads of the lower-level index blocks when the read-before was
insufficient to satisfy such reads).

Since remoteReadHandle already has a buffer for read-ahead, we utilize
it for this read-before buffering pattern too.

While here, we bump up the read-ahead size for compactions to 8MB, given
the lower concurrency (and thereby higher tolerance to memory usage).

Informs cockroachdb#2328

cockroach-teamcity: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/pebble/3608)
<!-- Reviewable:end -->


Patch:
diff --git a/objstorage/objstorage.go b/objstorage/objstorage.go
index d03be8c8..556e2ea4 100644
--- a/objstorage/objstorage.go
+++ b/objstorage/objstorage.go
@@ -38,9 +38,38 @@ type Readable interface {
 	// The ReadHandle must be closed before the Readable is closed.
 	//
 	// Multiple separate ReadHandles can be used.
-	NewReadHandle(ctx context.Context) ReadHandle
+	NewReadHandle(ctx context.Context, readBeforeSize ReadBeforeSize) ReadHandle
 }
 
+// ReadBeforeSize specifies whether the first read should read additional
+// bytes before the offset, and how big the overall read should be. This is
+// just a suggestion that the callee can ignore (and does ignore in
+// fileReadable).
+//
+// When 0, the first read will only read what it is asked to read, say n
+// bytes. When it is a value b > 0, if b > n, then the read will be padded by
+// an additional b-n bytes to the left, resulting in an overall read size of
+// b. This behavior is akin to what the read-ahead implementation does -- when
+// the n bytes are not buffered, and there is read-ahead of b > n, the read
+// length is b bytes.
+type ReadBeforeSize int64
+
+const (
+	// NoReadBefore specifies no read-before.
+	NoReadBefore ReadBeforeSize = 0
+	// ReadBeforeForNewReader is used for a new Reader reading the footer,
+	// metaindex, properties. 32KB is unnecessarily large, but it is still small
+	// when considering remote object storage.
+	ReadBeforeForNewReader = 32 * 1024
+	// ReadBeforeForIndexAndFilter is used for an iterator reading the top-level
+	// index, filter and second-level index blocks.
+	//
+	// Consider a 128MB sstable with 32KB blocks, so 4K blocks. Say keys are
+	// ~100 bytes, then the size of the index blocks is ~400KB. 512KB is a bit
+	// bigger, and not too large to be a memory concern.
+	ReadBeforeForIndexAndFilter = 512 * 1024
+)
+
 // ReadHandle is used to perform reads that are related and might benefit from
 // optimizations like read-ahead.
 type ReadHandle interface {
@@ -347,7 +376,7 @@ type RemoteObjectToAttach struct {
 
 // Copy copies the specified range from the input to the output.
 func Copy(ctx context.Context, in Readable, out Writable, offset, length uint64) error {
-	r := in.NewReadHandle(ctx)
+	r := in.NewReadHandle(ctx, NoReadBefore)
 	r.SetupForCompaction()
 	buf := make([]byte, 256<<10)
 	end := offset + length
diff --git a/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go b/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go
index d12bf8e0..76cb9583 100644
--- a/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go
+++ b/objstorage/objstorageprovider/objiotracing/obj_io_tracing_on.go
@@ -171,11 +171,13 @@ func (r *readable) Size() int64 {
 }
 
 // NewReadHandle is part of the objstorage.Readable interface.
-func (r *readable) NewReadHandle(ctx context.Context) objstorage.ReadHandle {
+func (r *readable) NewReadHandle(
+	ctx context.Context, readBeforeSize objstorage.ReadBeforeSize,
+) objstorage.ReadHandle {
 	// It's safe to get the tracer from the generator without the mutex since it never changes.
 	t := r.mu.g.t
 	return &readHandle{
-		rh:       r.r.NewReadHandle(ctx),
+		rh:       r.r.NewReadHandle(ctx, readBeforeSize),
 		fileNum:  r.fileNum,
 		handleID: t.handleID.Add(1),
 		g:        makeEventGenerator(ctx, t),
diff --git a/objstorage/objstorageprovider/remote_readable.go b/objstorage/objstorageprovider/remote_readable.go
index 8f001539..1780ba51 100644
--- a/objstorage/objstorageprovider/remote_readable.go
+++ b/objstorage/objstorageprovider/remote_readable.go
@@ -7,6 +7,7 @@ package objstorageprovider
 import (
 	"context"
 	"io"
+	"sync"
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/objstorage"
@@ -24,8 +25,15 @@ func NewRemoteReadable(objReader remote.ObjectReader, size int64) objstorage.Rea
 
 const remoteMaxReadaheadSize = 1024 * 1024 /* 1MB */
 
+// Number of concurrent compactions is bounded and significantly lower than
+// the number of concurrent queries, and compactions consume reads from a few
+// levels, so there is no risk of high memory usage due to a higher readahead
+// size. So set this higher than remoteMaxReadaheadSize
+const remoteReadaheadSizeForCompaction = 8 * 1024 * 1024 /* 8MB */
+
 // remoteReadable is a very simple implementation of Readable on top of the
-// ReadCloser returned by remote.Storage.CreateObject.
+// remote.ObjectReader returned by remote.Storage.ReadObject. It is stateless
+// and can be called concurrently.
 type remoteReadable struct {
 	objReader remote.ObjectReader
 	size      int64
@@ -75,17 +83,73 @@ func (r *remoteReadable) Size() int64 {
 	return r.size
 }
 
-func (r *remoteReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
-	// TODO(radu): use a pool.
-	rh := &remoteReadHandle{readable: r}
-	rh.readahead.state = makeReadaheadState(remoteMaxReadaheadSize)
+// TODO(sumeer): both readBeforeSize and ReadHandle.SetupForCompaction are
+// initial configuration of a ReadHandle. So they should both be passed as
+// Options to NewReadHandle. But currently the latter is a separate method.
+// This is because of how the sstable.Reader calls setupForCompaction on the
+// iterators after they are constructed. Consider fixing this oddity.
+
+func (r *remoteReadable) NewReadHandle(
+	ctx context.Context, readBeforeSize objstorage.ReadBeforeSize,
+) objstorage.ReadHandle {
+	rh := remoteReadHandlePool.Get().(*remoteReadHandle)
+	*rh = remoteReadHandle{readable: r, readBeforeSize: readBeforeSize}
+	rh.readAheadState = makeReadaheadState(remoteMaxReadaheadSize)
 	return rh
 }
 
+// TODO(sumeer): add test for remoteReadHandle.
+
+// remoteReadHandle supports doing larger reads, and buffering the additional
+// data, to serve future reads. It is not thread-safe. There are two kinds of
+// larger reads (a) read-ahead (for sequential data reads), (b) read-before,
+// for non-data reads.
+//
+// For both (a) and (b), the goal is to reduce the number of reads since
+// remote read latency and cost can be high. We have to balance this with
+// buffers consuming too much memory, since there can be a large number of
+// iterators holding remoteReadHandles open for every level.
+//
+// For (b) we have to two use-cases:
+//
+//   - When a sstable.Reader is opened, it needs to read the footer, metaindex
+//     block and meta properties block. It starts by reading the footer which is
+//     at the end of the table and then proceeds to read the other two. Instead
+//     of doing 3 tiny reads, we would like to do one read.
+//
+//   - When a single-level or two-level iterator is opened, it reads the
+//     (top-level) index block first. When the iterator is used, it will
+//     typically follow this by reading the filter block (since SeeKPrefixGE is
+//     common in CockroachDB). For a two-level iterator it will also read the
+//     lower-level index blocks which are after the filter block and before the
+//     top-level index block. It would be ideal if reading the top-level index
+//     block read enough to include the filter block. And for two-level
+//     iterators this would also include the lower-level index blocks.
+//
+// In both use-cases we want the first read from the remoteReadable to do a
+// larger read, and read bytes earlier than the requested read, hence
+// "read-before". Subsequent reads from the remoteReadable can use the usual
+// readahead logic (for the second use-case above, this can help with
+// sequential reads of the lower-level index blocks when the read-before was
+// insufficient to satisfy such reads). In the first use-case, the block cache
+// is not used. In the second use-case, the block cache is used, and if the
+// first read, which reads the top-level index, has a cache hit, we do not do
+// any read-before, since we assume that with some locality in the workload
+// the other reads will also have a cache hit (it is also messier code to try
+// to preserve some read-before).
+//
+// Note that both use-cases can often occur near each other if there is enough
+// locality of access, in which case table cache and block cache misses are
+// mainly happening for new sstables created by compactions -- in this case a
+// user-facing read will cause a table cache miss and a new sstable.Reader to
+// be created, followed by an iterator creation. We don't currently combine
+// the reads across the Reader and the iterator creation, since the code
+// structure is not simple enough, but we could consider that in the future.
 type remoteReadHandle struct {
-	readable  *remoteReadable
-	readahead struct {
-		state  readaheadState
+	readable       *remoteReadable
+	readBeforeSize objstorage.ReadBeforeSize
+	readAheadState readaheadState
+	buffered       struct {
 		data   []byte
 		offset int64
 	}
@@ -94,14 +158,41 @@ type remoteReadHandle struct {
 
 var _ objstorage.ReadHandle = (*remoteReadHandle)(nil)
 
+var remoteReadHandlePool = sync.Pool{
+	New: func() interface{} {
+		return &remoteReadHandle{}
+	},
+}
+
 // ReadAt is part of the objstorage.ReadHandle interface.
 func (r *remoteReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	var extraBytesBefore int64
+	if r.readBeforeSize > 0 {
+		if int(r.readBeforeSize) > len(p) {
+			extraBytesBefore = min(int64(int(r.readBeforeSize)-len(p)), offset)
+		}
+		// Only first read uses read-before.
+		r.readBeforeSize = 0
+	}
 	readaheadSize := r.maybeReadahead(offset, len(p))
 
-	// Check if we already have the data from a previous read-ahead.
-	if rhSize := int64(len(r.readahead.data)); rhSize > 0 {
-		if r.readahead.offset <= offset && r.readahead.offset+rhSize > offset {
-			n := copy(p, r.readahead.data[offset-r.readahead.offset:])
+	// Prefer read-before to read-ahead since only first call does read-before.
+	// Also, since this is the first call, the buffer must be empty.
+	if extraBytesBefore > 0 {
+		r.buffered.offset = offset - extraBytesBefore
+		err := r.readToBuffer(ctx, offset-extraBytesBefore, len(p)+int(extraBytesBefore))
+		if err != nil {
+			return err
+		}
+		copy(p, r.buffered.data[int(extraBytesBefore):])
+		return nil
+	}
+	// Check if we already have the data from a previous read-ahead/read-before.
+	if rhSize := int64(len(r.buffered.data)); rhSize > 0 {
+		// We only consider the case where we have a prefix of the needed data. We
+		// could enhance this to utilize a suffix of the needed data.
+		if r.buffered.offset <= offset && r.buffered.offset+rhSize > offset {
+			n := copy(p, r.buffered.data[offset-r.buffered.offset:])
 			if n == len(p) {
 				// All data was available.
 				return nil
@@ -123,20 +214,10 @@ func (r *remoteReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) e
 				return io.EOF
 			}
 		}
-		r.readahead.offset = offset
-		// TODO(radu): we need to somehow account for this memory.
-		if cap(r.readahead.data) >= readaheadSize {
-			r.readahead.data = r.readahead.data[:readaheadSize]
-		} else {
-			r.readahead.data = make([]byte, readaheadSize)
-		}
-
-		if err := r.readable.readInternal(ctx, r.readahead.data, offset, r.forCompaction); err != nil {
-			// Make sure we don't treat the data as valid next time.
-			r.readahead.data = r.readahead.data[:0]
+		if err := r.readToBuffer(ctx, offset, readaheadSize); err != nil {
 			return err
 		}
-		copy(p, r.readahead.data)
+		copy(p, r.buffered.data)
 		return nil
 	}
 
@@ -145,15 +226,32 @@ func (r *remoteReadHandle) ReadAt(ctx context.Context, p []byte, offset int64) e
 
 func (r *remoteReadHandle) maybeReadahead(offset int64, len int) int {
 	if r.forCompaction {
-		return remoteMaxReadaheadSize
+		return remoteReadaheadSizeForCompaction
 	}
-	return int(r.readahead.state.maybeReadahead(offset, int64(len)))
+	return int(r.readAheadState.maybeReadahead(offset, int64(len)))
+}
+
+func (r *remoteReadHandle) readToBuffer(ctx context.Context, offset int64, length int) error {
+	r.buffered.offset = offset
+	// TODO(radu): we need to somehow account for this memory.
+	if cap(r.buffered.data) >= length {
+		r.buffered.data = r.buffered.data[:length]
+	} else {
+		r.buffered.data = make([]byte, length)
+	}
+	if err := r.readable.readInternal(
+		ctx, r.buffered.data, r.buffered.offset, r.forCompaction); err != nil {
+		// Make sure we don't treat the data as valid next time.
+		r.buffered.data = r.buffered.data[:0]
+		return err
+	}
+	return nil
 }
 
 // Close is part of the objstorage.ReadHandle interface.
 func (r *remoteReadHandle) Close() error {
-	r.readable = nil
-	r.readahead.data = nil
+	*r = remoteReadHandle{}
+	remoteReadHandlePool.Put(r)
 	return nil
 }
 
@@ -165,6 +263,9 @@ func (r *remoteReadHandle) SetupForCompaction() {
 // RecordCacheHit is part of the objstorage.ReadHandle interface.
 func (r *remoteReadHandle) RecordCacheHit(_ context.Context, offset, size int64) {
 	if !r.forCompaction {
-		r.readahead.state.recordCacheHit(offset, size)
+		r.readAheadState.recordCacheHit(offset, size)
+	}
+	if r.readBeforeSize > 0 {
+		r.readBeforeSize = 0
 	}
 }
diff --git a/objstorage/objstorageprovider/vfs_readable.go b/objstorage/objstorageprovider/vfs_readable.go
index 93624646..ecf73838 100644
--- a/objstorage/objstorageprovider/vfs_readable.go
+++ b/objstorage/objstorageprovider/vfs_readable.go
@@ -74,7 +74,9 @@ func (r *fileReadable) Size() int64 {
 }
 
 // NewReadHandle is part of the objstorage.Readable interface.
-func (r *fileReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+func (r *fileReadable) NewReadHandle(
+	ctx context.Context, readBeforeSize objstorage.ReadBeforeSize,
+) objstorage.ReadHandle {
 	rh := readHandlePool.Get().(*vfsReadHandle)
 	rh.r = r
 	rh.rs = makeReadaheadState(fileMaxReadaheadSize)
@@ -205,12 +207,15 @@ func (rh *PreallocatedReadHandle) Close() error {
 // (currently this happens if we are reading from a local file).
 // The returned handle still needs to be closed.
 func UsePreallocatedReadHandle(
-	ctx context.Context, readable objstorage.Readable, rh *PreallocatedReadHandle,
+	ctx context.Context,
+	readable objstorage.Readable,
+	readBeforeSize objstorage.ReadBeforeSize,
+	rh *PreallocatedReadHandle,
 ) objstorage.ReadHandle {
 	if r, ok := readable.(*fileReadable); ok {
 		// See fileReadable.NewReadHandle.
 		rh.vfsReadHandle = vfsReadHandle{r: r}
 		return rh
 	}
-	return readable.NewReadHandle(ctx)
+	return readable.NewReadHandle(ctx, readBeforeSize)
 }
diff --git a/sstable/copier.go b/sstable/copier.go
index 207b427f..8c362c09 100644
--- a/sstable/copier.go
+++ b/sstable/copier.go
@@ -11,6 +11,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/bytealloc"
 	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
 )
 
 // CopySpan produces a copy of a approximate subset of an input sstable.
@@ -88,11 +89,23 @@ func CopySpan(
 		return 0, base.AssertionFailedf("cannot CopySpan sstables with value blocks or range keys")
 	}
 
+	var preallocRH objstorageprovider.PreallocatedReadHandle
+	// ReadBeforeForIndexAndFilter attempts to read the top-level index, filter
+	// and lower-level index blocks with one read.
+	rh := objstorageprovider.UsePreallocatedReadHandle(
+		ctx, r.readable, objstorage.ReadBeforeForIndexAndFilter, &preallocRH)
+	defer rh.Close()
+	indexH, err := r.readIndex(ctx, rh, nil, nil)
+	if err != nil {
+		return 0, err
+	}
+	defer indexH.Release()
+
 	// Set the filter block to be copied over if it exists. It will return false
 	// positives for keys in blocks of the original file that we don't copy, but
 	// filters can always have false positives, so this is fine.
 	if r.tableFilter != nil {
-		filterBlock, err := r.readFilter(ctx, nil, nil)
+		filterBlock, err := r.readFilter(ctx, rh, nil, nil)
 		if err != nil {
 			return 0, errors.Wrap(err, "reading filter")
 		}
@@ -123,7 +136,7 @@ func CopySpan(
 	}
 
 	// Find the blocks that intersect our span.
-	blocks, err := intersectingIndexEntries(ctx, r, start, end)
+	blocks, err := intersectingIndexEntries(ctx, r, rh, indexH, start, end)
 	if err != nil {
 		return 0, err
 	}
@@ -195,25 +208,18 @@ type indexEntry struct {
 // keys contained by [start, end), i.e. the subset of the sst's index that
 // intersects the provided span.
 func intersectingIndexEntries(
-	ctx context.Context, r *Reader, start, end InternalKey,
+	ctx context.Context,
+	r *Reader,
+	rh objstorage.ReadHandle,
+	indexH bufferHandle,
+	start, end InternalKey,
 ) ([]indexEntry, error) {
-	indexH, err := r.readIndex(ctx, nil, nil)
-	if err != nil {
-		return nil, err
-	}
-	defer indexH.Release()
 	top, err := newBlockIter(r.Compare, r.Split, indexH.Get(), NoTransforms)
 	if err != nil {
 		return nil, err
 	}
 	defer top.Close()
 
-	var rh objstorage.ReadHandle
-	if r.Properties.IndexType == twoLevelIndex {
-		rh = r.readable.NewReadHandle(ctx)
-		defer rh.Close()
-	}
-
 	var alloc bytealloc.A
 	res := make([]indexEntry, 0, r.Properties.NumDataBlocks)
 	for kv := top.SeekGE(start.UserKey, base.SeekGEFlagsNone); kv != nil; kv = top.Next() {
diff --git a/sstable/reader.go b/sstable/reader.go
index f7592dc6..084ae0a1 100644
--- a/sstable/reader.go
+++ b/sstable/reader.go
@@ -25,6 +25,7 @@ import (
 	"github.com/cockroachdb/pebble/internal/keyspan"
 	"github.com/cockroachdb/pebble/internal/private"
 	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
 )
 
@@ -503,17 +504,23 @@ func (i *rangeKeyFragmentBlockIter) Close() error {
 }
 
 func (r *Reader) readIndex(
-	ctx context.Context, stats *base.InternalIteratorStats, iterStats *iterStatsAccumulator,
+	ctx context.Context,
+	readHandle objstorage.ReadHandle,
+	stats *base.InternalIteratorStats,
+	iterStats *iterStatsAccumulator,
 ) (bufferHandle, error) {
 	ctx = objiotracing.WithBlockType(ctx, objiotracing.MetadataBlock)
-	return r.readBlock(ctx, r.indexBH, nil, nil, stats, iterStats, nil /* buffer pool */)
+	return r.readBlock(ctx, r.indexBH, nil, readHandle, stats, iterStats, nil /* buffer pool */)
 }
 
 func (r *Reader) readFilter(
-	ctx context.Context, stats *base.InternalIteratorStats, iterStats *iterStatsAccumulator,
+	ctx context.Context,
+	readHandle objstorage.ReadHandle,
+	stats *base.InternalIteratorStats,
+	iterStats *iterStatsAccumulator,
 ) (bufferHandle, error) {
 	ctx = objiotracing.WithBlockType(ctx, objiotracing.FilterBlock)
-	return r.readBlock(ctx, r.filterBH, nil /* transform */, nil /* readHandle */, stats, iterStats, nil /* buffer pool */)
+	return r.readBlock(ctx, r.filterBH, nil /* transform */, readHandle, stats, iterStats, nil /* buffer pool */)
 }
 
 func (r *Reader) readRangeDel(
@@ -767,7 +774,7 @@ func (r *Reader) transformRangeDelV1(b []byte) ([]byte, error) {
 	return rangeDelBlock.finish(), nil
 }
 
-func (r *Reader) readMetaindex(metaindexBH BlockHandle) error {
+func (r *Reader) readMetaindex(metaindexBH BlockHandle, readHandle objstorage.ReadHandle) error {
 	// We use a BufferPool when reading metaindex blocks in order to avoid
 	// populating the block cache with these blocks. In heavy-write workloads,
 	// especially with high compaction concurrency, new tables may be created
@@ -782,7 +789,7 @@ func (r *Reader) readMetaindex(metaindexBH BlockHandle) error {
 	defer r.metaBufferPool.Release()
 
 	b, err := r.readBlock(
-		context.Background(), metaindexBH, nil /* transform */, nil /* readHandle */, nil, /* stats */
+		context.Background(), metaindexBH, nil /* transform */, readHandle, nil, /* stats */
 		nil /* iterStats */, &r.metaBufferPool)
 	if err != nil {
 		return err
@@ -826,7 +833,7 @@ func (r *Reader) readMetaindex(metaindexBH BlockHandle) error {
 
 	if bh, ok := meta[metaPropertiesName]; ok {
 		b, err = r.readBlock(
-			context.Background(), bh, nil /* transform */, nil /* readHandle */, nil, /* stats */
+			context.Background(), bh, nil /* transform */, readHandle, nil, /* stats */
 			nil /* iterStats */, nil /* buffer pool */)
 		if err != nil {
 			return err
@@ -900,7 +907,7 @@ func (r *Reader) Layout() (*Layout, error) {
 		Format:     r.tableFormat,
 	}
 
-	indexH, err := r.readIndex(context.Background(), nil, nil)
+	indexH, err := r.readIndex(context.Background(), nil, nil, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -1017,7 +1024,7 @@ func (r *Reader) ValidateBlockChecksums() error {
 
 	// Check all blocks sequentially. Make use of read-ahead, given we are
 	// scanning the entire file from start to end.
-	rh := r.readable.NewReadHandle(context.TODO())
+	rh := r.readable.NewReadHandle(context.TODO(), objstorage.NoReadBefore)
 	defer rh.Close()
 
 	for _, bh := range blocks {
@@ -1062,7 +1069,7 @@ func (r *Reader) EstimateDiskUsage(start, end []byte) (uint64, error) {
 		return 0, r.err
 	}
 
-	indexH, err := r.readIndex(context.Background(), nil, nil)
+	indexH, err := r.readIndex(context.Background(), nil, nil, nil)
 	if err != nil {
 		return 0, err
 	}
@@ -1214,15 +1221,21 @@ func NewReader(f objstorage.Readable, o ReaderOptions, extraOpts ...ReaderOption
 		r.cacheID = r.opts.Cache.NewID()
 	}
 
-	footer, err := readFooter(f)
+	var preallocRH objstorageprovider.PreallocatedReadHandle
+	ctx := context.TODO()
+	rh := objstorageprovider.UsePreallocatedReadHandle(
+		ctx, r.readable, objstorage.ReadBeforeForNewReader, &preallocRH)
+	defer rh.Close()
+
+	footer, err := readFooter(ctx, f, rh)
 	if err != nil {
 		r.err = err
 		return nil, r.Close()
 	}
 	r.checksumType = footer.checksum
 	r.tableFormat = footer.format
-	// Read the metaindex.
-	if err := r.readMetaindex(footer.metaindexBH); err != nil {
+	// Read the metaindex and properties blocks.
+	if err := r.readMetaindex(footer.metaindexBH, rh); err != nil {
 		r.err = err
 		return nil, r.Close()
 	}
@@ -1318,7 +1331,9 @@ func (s *simpleReadable) Size() int64 {
 }
 
 // NewReaddHandle is part of the objstorage.Readable interface.
-func (s *simpleReadable) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+func (s *simpleReadable) NewReadHandle(
+	ctx context.Context, readBeforeSize objstorage.ReadBeforeSize,
+) objstorage.ReadHandle {
 	return &s.rh
 }
 
diff --git a/sstable/reader_iter_single_lvl.go b/sstable/reader_iter_single_lvl.go
index 3c0a7b31..6f86c53f 100644
--- a/sstable/reader_iter_single_lvl.go
+++ b/sstable/reader_iter_single_lvl.go
@@ -37,11 +37,13 @@ type singleLevelIterator struct {
 	vState *virtualState
 	// endKeyInclusive is set to force the iterator to treat the upper field as
 	// inclusive while iterating instead of exclusive.
-	endKeyInclusive bool
-	index           blockIter
-	data            blockIter
-	dataRH          objstorage.ReadHandle
-	dataRHPrealloc  objstorageprovider.PreallocatedReadHandle
+	endKeyInclusive       bool
+	index                 blockIter
+	indexFilterRH         objstorage.ReadHandle
+	indexFilterRHPrealloc objstorageprovider.PreallocatedReadHandle
+	data                  blockIter
+	dataRH                objstorage.ReadHandle
+	dataRHPrealloc        objstorageprovider.PreallocatedReadHandle
 	// dataBH refers to the last data block that the iterator considered
 	// loading. It may not actually have loaded the block, due to an error or
 	// because it was considered irrelevant.
@@ -194,7 +196,9 @@ func (i *singleLevelIterator) init(
 		return r.err
 	}
 	i.iterStats.init(categoryAndQoS, statsCollector)
-	indexH, err := r.readIndex(ctx, stats, &i.iterStats)
+	i.indexFilterRH = objstorageprovider.UsePreallocatedReadHandle(
+		ctx, r.readable, objstorage.ReadBeforeForIndexAndFilter, &i.indexFilterRHPrealloc)
+	indexH, err := r.readIndex(ctx, i.indexFilterRH, stats, &i.iterStats)
 	if err != nil {
 		return err
 	}
@@ -220,7 +224,7 @@ func (i *singleLevelIterator) init(
 		_ = i.index.Close()
 		return err
 	}
-	i.dataRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, &i.dataRHPrealloc)
+	i.dataRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, objstorage.NoReadBefore, &i.dataRHPrealloc)
 	if r.tableFormat >= TableFormatPebblev3 {
 		if r.Properties.NumValueBlocks > 0 {
 			// NB: we cannot avoid this ~248 byte allocation, since valueBlockReader
@@ -239,7 +243,7 @@ func (i *singleLevelIterator) init(
 				stats:  stats,
 			}
 			i.data.lazyValueHandling.vbr = i.vbReader
-			i.vbRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, &i.vbRHPrealloc)
+			i.vbRH = objstorageprovider.UsePreallocatedReadHandle(ctx, r.readable, objstorage.NoReadBefore, &i.vbRHPrealloc)
 		}
 		i.data.lazyValueHandling.hasValuePrefix = true
 	}
@@ -783,7 +787,7 @@ func (i *singleLevelIterator) seekPrefixGE(
 		i.lastBloomFilterMatched = false
 		// Check prefix bloom filter.
 		var dataH bufferHandle
-		dataH, i.err = i.reader.readFilter(i.ctx, i.stats, &i.iterStats)
+		dataH, i.err = i.reader.readFilter(i.ctx, i.indexFilterRH, i.stats, &i.iterStats)
 		if i.err != nil {
 			i.data.invalidate()
 			return nil
@@ -1448,6 +1452,10 @@ func (i *singleLevelIterator) Close() error {
 	}
 	err = firstError(err, i.data.Close())
 	err = firstError(err, i.index.Close())
+	if i.indexFilterRH != nil {
+		err = firstError(err, i.indexFilterRH.Close())
+		i.indexFilterRH = nil
+	}
 	if i.dataRH != nil {
 		err = firstError(err, i.dataRH.Close())
 		i.dataRH = nil
diff --git a/sstable/reader_iter_two_lvl.go b/sstable/reader_iter_two_lvl.go
index f475fa03..661f8537 100644
--- a/sstable/reader_iter_two_lvl.go
+++ b/sstable/reader_iter_two_lvl.go
@@ -11,6 +11,8 @@ import (
 
 	"github.com/cockroachdb/pebble/internal/base"
 	"github.com/cockroachdb/pebble/internal/invariants"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/cockroachdb/pebble/objstorage/objstorageprovider"
 	"github.com/cockroachdb/pebble/objstorage/objstorageprovider/objiotracing"
 )
 
@@ -58,7 +60,7 @@ func (i *twoLevelIterator) loadIndex(dir int8) loadBlockResult {
 	}
 	ctx := objiotracing.WithBlockType(i.ctx, objiotracing.MetadataBlock)
 	indexBlock, err := i.reader.readBlock(
-		ctx, bhp.BlockHandle, nil /* transform */, nil /* readHandle */, i.stats, &i.iterStats, i.bufferPool)
+		ctx, bhp.BlockHandle, nil /* transform */, i.indexFilterRH, i.stats, &i.iterStats, i.bufferPool)
 	if err != nil {
 		i.err = err
 		return loadBlockFailed
@@ -152,7 +154,9 @@ func (i *twoLevelIterator) init(
 		return r.err
 	}
 	i.iterStats.init(categoryAndQoS, statsCollector)
-	topLevelIndexH, err := r.readIndex(ctx, stats, &i.iterStats)
+	i.indexFilterRH = objstorageprovider.UsePreallocatedReadHandle(
+		ctx, r.readable, objstorage.ReadBeforeForIndexAndFilter, &i.indexFilterRHPrealloc)
+	topLevelIndexH, err := r.readIndex(ctx, i.indexFilterRH, stats, &i.iterStats)
 	if err != nil {
 		return err
 	}
@@ -179,7 +183,8 @@ func (i *twoLevelIterator) init(
 		_ = i.topLevelIndex.Close()
 		return err
 	}
-	i.dataRH = r.readable.NewReadHandle(ctx)
+	i.dataRH = objstorageprovider.UsePreallocatedReadHandle(
+		ctx, r.readable, objstorage.NoReadBefore, &i.dataRHPrealloc)
 	if r.tableFormat >= TableFormatPebblev3 {
 		if r.Properties.NumValueBlocks > 0 {
 			i.vbReader = &valueBlockReader{
@@ -189,7 +194,8 @@ func (i *twoLevelIterator) init(
 				stats:  stats,
 			}
 			i.data.lazyValueHandling.vbr = i.vbReader
-			i.vbRH = r.readable.NewReadHandle(ctx)
+			i.vbRH = objstorageprovider.UsePreallocatedReadHandle(
+				ctx, r.readable, objstorage.NoReadBefore, &i.vbRHPrealloc)
 		}
 		i.data.lazyValueHandling.hasValuePrefix = true
 	}
@@ -385,7 +391,7 @@ func (i *twoLevelIterator) SeekPrefixGE(
 		}
 		i.lastBloomFilterMatched = false
 		var dataH bufferHandle
-		dataH, i.err = i.reader.readFilter(i.ctx, i.stats, &i.iterStats)
+		dataH, i.err = i.reader.readFilter(i.ctx, i.indexFilterRH, i.stats, &i.iterStats)
 		if i.err != nil {
 			i.data.invalidate()
 			return nil
@@ -968,6 +974,10 @@ func (i *twoLevelIterator) Close() error {
 	err = firstError(err, i.data.Close())
 	err = firstError(err, i.index.Close())
 	err = firstError(err, i.topLevelIndex.Close())
+	if i.indexFilterRH != nil {
+		err = firstError(err, i.indexFilterRH.Close())
+		i.indexFilterRH = nil
+	}
 	if i.dataRH != nil {
 		err = firstError(err, i.dataRH.Close())
 		i.dataRH = nil
diff --git a/sstable/suffix_rewriter.go b/sstable/suffix_rewriter.go
index 1ffa0c56..76ad840d 100644
--- a/sstable/suffix_rewriter.go
+++ b/sstable/suffix_rewriter.go
@@ -575,6 +575,8 @@ func (m *memReader) Size() int64 {
 }
 
 // NewReadHandle is part of objstorage.Readable.
-func (m *memReader) NewReadHandle(_ context.Context) objstorage.ReadHandle {
+func (m *memReader) NewReadHandle(
+	ctx context.Context, readBeforeSize objstorage.ReadBeforeSize,
+) objstorage.ReadHandle {
 	return &m.rh
 }
diff --git a/sstable/table.go b/sstable/table.go
index 2f36f307..a64ed9ca 100644
--- a/sstable/table.go
+++ b/sstable/table.go
@@ -324,7 +324,10 @@ type footer struct {
 	footerBH    BlockHandle
 }
 
-func readFooter(f objstorage.Readable) (footer, error) {
+// readHandle is optional.
+func readFooter(
+	ctx context.Context, f objstorage.Readable, readHandle objstorage.ReadHandle,
+) (footer, error) {
 	var footer footer
 	size := f.Size()
 	if size < minFooterLen {
@@ -337,7 +340,13 @@ func readFooter(f objstorage.Readable) (footer, error) {
 		off = 0
 		buf = buf[:size]
 	}
-	if err := f.ReadAt(context.TODO(), buf, off); err != nil {
+	var err error
+	if readHandle != nil {
+		err = readHandle.ReadAt(ctx, buf, off)
+	} else {
+		err = f.ReadAt(ctx, buf, off)
+	}
+	if err != nil {
 		return footer, errors.Wrap(err, "pebble/table: invalid table (could not read footer)")
 	}
 
diff --git a/tool/db_io_bench.go b/tool/db_io_bench.go
index 3d750782..20cd08f2 100644
--- a/tool/db_io_bench.go
+++ b/tool/db_io_bench.go
@@ -248,7 +248,7 @@ func performIOs(readables []objstorage.Readable, ios []benchIO) error {
 	ctx := context.Background()
 	rh := make([]objstorage.ReadHandle, len(readables))
 	for i := range rh {
-		rh[i] = readables[i].NewReadHandle(ctx)
+		rh[i] = readables[i].NewReadHandle(ctx, objstorage.NoReadBefore)
 	}
 	defer func() {
 		for i := range rh {


Test patch:
diff --git a/objstorage/objstorageprovider/provider_test.go b/objstorage/objstorageprovider/provider_test.go
index 1a791d4f..1999ec63 100644
--- a/objstorage/objstorageprovider/provider_test.go
+++ b/objstorage/objstorageprovider/provider_test.go
@@ -181,7 +181,7 @@ func TestProvider(t *testing.T) {
 				if err != nil {
 					return err.Error()
 				}
-				rh := r.NewReadHandle(ctx)
+				rh := r.NewReadHandle(ctx, objstorage.NoReadBefore)
 				if forCompaction {
 					rh.SetupForCompaction()
 				}
diff --git a/objstorage/objstorageprovider/remote_readable_test.go b/objstorage/objstorageprovider/remote_readable_test.go
new file mode 100644
index 00000000..721c2ff7
--- /dev/null
+++ b/objstorage/objstorageprovider/remote_readable_test.go
@@ -0,0 +1,108 @@
+// Copyright 2023 The LevelDB-Go and Pebble Authors. All rights reserved. Use
+// of this source code is governed by a BSD-style license that can be found in
+// the LICENSE file.
+
+package objstorageprovider
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"math/rand"
+	"strings"
+	"testing"
+
+	"github.com/cockroachdb/datadriven"
+	"github.com/cockroachdb/pebble/objstorage"
+	"github.com/stretchr/testify/require"
+)
+
+type testObjectReader struct {
+	buf []byte
+	b   strings.Builder
+}
+
+func (r *testObjectReader) init(size int) {
+	r.buf = make([]byte, size)
+	const letters = "abcdefghijklmnopqrstuvwxyz"
+	const lettersLen = len(letters)
+	for i := 0; i < len(r.buf); i++ {
+		r.buf[i] = letters[rand.Intn(lettersLen)]
+	}
+}
+
+func (r *testObjectReader) ReadAt(ctx context.Context, p []byte, offset int64) error {
+	fmt.Fprintf(&r.b, "ReadAt(len=%d, offset=%d)\n", len(p), offset)
+	limit := int(offset) + len(p)
+	if limit > len(r.buf) {
+		return io.EOF
+	}
+	copy(p, r.buf[offset:limit])
+	return nil
+}
+
+func (r *testObjectReader) Close() error {
+	fmt.Fprintf(&r.b, "Close()\n")
+	return nil
+}
+
+func TestRemoteReadHandle(t *testing.T) {
+	var or testObjectReader
+	var rr *remoteReadable
+	var rh objstorage.ReadHandle
+	defer func() {
+		if rh != nil {
+			require.NoError(t, rh.Close())
+		}
+		if rr != nil {
+			require.NoError(t, rr.Close())
+		}
+	}()
+	datadriven.RunTest(t, "testdata/remote_read_handle", func(t *testing.T, d *datadriven.TestData) string {
+		switch d.Cmd {
+		case "init-readable":
+			if rr != nil {
+				require.NoError(t, rr.Close())
+			}
+			var size int64
+			d.ScanArgs(t, "size", &size)
+			or.init(int(size))
+			rr = &remoteReadable{
+				objReader: &or,
+				size:      size,
+			}
+			return ""
+
+		case "new-read-handle":
+			if rh != nil {
+				require.NoError(t, rh.Close())
+			}
+			var readBeforeSize int
+			d.ScanArgs(t, "read-before-size", &readBeforeSize)
+			rh = rr.NewReadHandle(context.Background(), objstorage.ReadBeforeSize(readBeforeSize))
+			if d.HasArg("setup-for-compaction") {
+				rh.SetupForCompaction()
+			}
+			return ""
+
+		case "read":
+			var length int
+			d.ScanArgs(t, "len", &length)
+			b := make([]byte, length)
+			var offset int64
+			d.ScanArgs(t, "offset", &offset)
+			err := rh.ReadAt(context.Background(), b, offset)
+			if err != nil {
+				fmt.Fprintf(&or.b, "err: %s\n", err.Error())
+			} else {
+				require.Equal(t, string(or.buf[offset:int(offset)+length]), string(b))
+			}
+			str := or.b.String()
+			or.b.Reset()
+			return str
+
+		default:
+			return fmt.Sprintf("unknown command: %s", d.Cmd)
+		}
+	})
+}
diff --git a/objstorage/objstorageprovider/testdata/provider/shared_readahead b/objstorage/objstorageprovider/testdata/provider/shared_readahead
index 45e1c497..f2853a2c 100644
--- a/objstorage/objstorageprovider/testdata/provider/shared_readahead
+++ b/objstorage/objstorageprovider/testdata/provider/shared_readahead
@@ -74,7 +74,7 @@ read 1 for-compaction
 <remote> size of object "61a6-1-000001.sst.ref.1.000001": 0
 <remote> create reader for object "61a6-1-000001.sst": 2000000 bytes
 size: 2000000
-<remote> read object "61a6-1-000001.sst" at 0 (length 1048576)
+<remote> read object "61a6-1-000001.sst" at 0 (length 2000000)
 0 1000: ok (salt 1)
 1000 15000: ok (salt 1)
 16000 30000: ok (salt 1)
@@ -85,3 +85,24 @@ size: 2000000
 180000 10000: ok (salt 1)
 210000 30000: ok (salt 1)
 <remote> close reader for "61a6-1-000001.sst"
+
+# When reading for a compaction, we should be doing 8MB reads from the start.
+create 2 shared 2 15000000
+----
+<remote> create object "a629-1-000002.sst"
+<remote> close writer for "a629-1-000002.sst" after 15000000 bytes
+<remote> create object "a629-1-000002.sst.ref.1.000002"
+<remote> close writer for "a629-1-000002.sst.ref.1.000002" after 0 bytes
+
+read 2 for-compaction
+0 100000
+9000000 3000000
+----
+<remote> size of object "a629-1-000002.sst.ref.1.000002": 0
+<remote> create reader for object "a629-1-000002.sst": 15000000 bytes
+size: 15000000
+<remote> read object "a629-1-000002.sst" at 0 (length 8388608)
+0 100000: ok (salt 2)
+<remote> read object "a629-1-000002.sst" at 9000000 (length 6000000)
+9000000 3000000: ok (salt 2)
+<remote> close reader for "a629-1-000002.sst"
diff --git a/objstorage/objstorageprovider/testdata/remote_read_handle b/objstorage/objstorageprovider/testdata/remote_read_handle
new file mode 100644
index 00000000..08567a10
--- /dev/null
+++ b/objstorage/objstorageprovider/testdata/remote_read_handle
@@ -0,0 +1,68 @@
+init-readable size=100000
+----
+
+# remoteReadHandle with no read-before, but will read-ahead.
+new-read-handle read-before-size=0
+----
+
+read offset=50 len=100
+----
+ReadAt(len=100, offset=50)
+
+read offset=150 len=100
+----
+ReadAt(len=100, offset=150)
+
+# First read-ahead.
+read offset=250 len=100
+----
+ReadAt(len=65536, offset=250)
+
+read offset=350 len=100
+----
+
+# Read-ahead up to EOF.
+read offset=65000 len=10000
+----
+ReadAt(len=34214, offset=65786)
+
+# remoteReadHandle with read-before.
+new-read-handle read-before-size=2000
+----
+
+# Read-before happens.
+read offset=99000 len=100
+----
+ReadAt(len=2000, offset=97100)
+
+# Already buffered because of read-before.
+read offset=97300 len=300
+----
+
+# Partially in buffer, but need to read the remaining.
+read offset=99000 len=200
+----
+ReadAt(len=100, offset=99100)
+
+# Read-ahead up to EOF.
+read offset=99200 len=200
+----
+ReadAt(len=800, offset=99200)
+
+# remoteReadHandle with read-before and setup-for-compaction.
+new-read-handle read-before-size=1000 setup-for-compaction
+----
+
+# First read does read-before.
+read offset=99000 len=100
+----
+ReadAt(len=1000, offset=98100)
+
+# Next read does the maximum read-ahead, constrained by EOF.
+read offset=100 len=100
+----
+ReadAt(len=99900, offset=100)
+
+read offset=99000 len=2000
+----
+err: EOF
diff --git a/objstorage/test_utils.go b/objstorage/test_utils.go
index c16a73c8..928eeeb6 100644
--- a/objstorage/test_utils.go
+++ b/objstorage/test_utils.go
@@ -58,6 +58,6 @@ func (f *MemObj) Size() int64 {
 }
 
 // NewReadHandle is part of the Readable interface.
-func (f *MemObj) NewReadHandle(ctx context.Context) ReadHandle {
+func (f *MemObj) NewReadHandle(ctx context.Context, readBeforeSize ReadBeforeSize) ReadHandle {
 	return &NoopReadHandle{readable: f}
 }
diff --git a/sstable/block_property_test.go b/sstable/block_property_test.go
index f39d96ee..f7d8d1a4 100644
--- a/sstable/block_property_test.go
+++ b/sstable/block_property_test.go
@@ -981,7 +981,7 @@ func TestBlockProperties(t *testing.T) {
 
 				// Enumerate point key data blocks encoded into the index.
 				if f != nil {
-					indexH, err := r.readIndex(context.Background(), nil, nil)
+					indexH, err := r.readIndex(context.Background(), nil, nil, nil)
 					if err != nil {
 						return err.Error()
 					}
@@ -1347,7 +1347,7 @@ func runBlockPropertiesBuildCmd(td *datadriven.TestData) (r *Reader, out string)
 }
 
 func runBlockPropsCmd(r *Reader, td *datadriven.TestData) string {
-	bh, err := r.readIndex(context.Background(), nil, nil)
+	bh, err := r.readIndex(context.Background(), nil, nil, nil)
 	if err != nil {
 		return err.Error()
 	}
diff --git a/sstable/reader_test.go b/sstable/reader_test.go
index 817ba35d..87b25838 100644
--- a/sstable/reader_test.go
+++ b/sstable/reader_test.go
@@ -43,7 +43,7 @@ func (r *Reader) get(key []byte) (value []byte, err error) {
 	}
 
 	if r.tableFilter != nil {
-		dataH, err := r.readFilter(context.Background(), nil /* stats */, nil)
+		dataH, err := r.readFilter(context.Background(), nil, nil /* stats */, nil)
 		if err != nil {
 			return nil, err
 		}
@@ -700,7 +700,7 @@ func TestInvalidReader(t *testing.T) {
 }
 
 func indexLayoutString(t *testing.T, r *Reader) string {
-	indexH, err := r.readIndex(context.Background(), nil, nil)
+	indexH, err := r.readIndex(context.Background(), nil, nil, nil)
 	require.NoError(t, err)
 	defer indexH.Release()
 	var buf strings.Builder
diff --git a/sstable/table_test.go b/sstable/table_test.go
index 506a38be..afe9ae2a 100644
--- a/sstable/table_test.go
+++ b/sstable/table_test.go
@@ -586,7 +586,7 @@ func TestFooterRoundTrip(t *testing.T) {
 							readable, err := NewSimpleReadable(f)
 							require.NoError(t, err)
 
-							result, err := readFooter(readable)
+							result, err := readFooter(context.Background(), readable, nil)
 							require.NoError(t, err)
 							require.NoError(t, readable.Close())
 
@@ -639,7 +639,7 @@ func TestReadFooter(t *testing.T) {
 			readable, err := NewSimpleReadable(f)
 			require.NoError(t, err)
 
-			if _, err := readFooter(readable); err == nil {
+			if _, err := readFooter(context.Background(), readable, nil); err == nil {
 				t.Fatalf("expected %q, but found success", c.expected)
 			} else if !strings.Contains(err.Error(), c.expected) {
 				t.Fatalf("expected %q, but found %v", c.expected, err)