Repository: cockroachdb/pebble
Pull request #3753 (2024-07-12 16:15:40)

crl-release-23.1: sstable: trace/expensive log when sstable footer read is slow
backport https://github.com/cockroachdb/pebble/pull/3745

----

This is a manual backport of https://github.com/cockroachdb/pebble/pull/3745

Relates to #3728

cockroach-teamcity: 

<!-- Reviewable:start -->
This change is [<img src="https://reviewable.io/review_button.svg" height="34" align="absmiddle" alt="Reviewable"/>](https://reviewable.io/reviews/cockroachdb/pebble/3753)
<!-- Reviewable:end -->


Patch:
diff --git a/internal/base/logger.go b/internal/base/logger.go
index e4a2f44f..5348459b 100644
--- a/internal/base/logger.go
+++ b/internal/base/logger.go
@@ -85,11 +85,13 @@ func (b *InMemLogger) Fatalf(format string, args ...interface{}) {
 type LoggerAndTracer interface {
 	Logger
 	// Eventf formats and emits a tracing log, if tracing is enabled in the
-	// current context.
+	// current context. It can also emit to a regular log, if expensive
+	// logging is enabled.
 	Eventf(ctx context.Context, format string, args ...interface{})
-	// IsTracingEnabled returns true if tracing is enabled. It can be used as an
-	// optimization to avoid calling Eventf (which will be a noop when tracing
-	// is not enabled) to avoid the overhead of boxing the args.
+	// IsTracingEnabled returns true if tracing is enabled for this context,
+	// or expensive logging is enabled. It can be used as an optimization to
+	// avoid calling Eventf (which will be a noop when tracing or expensive
+	// logging is not enabled) to avoid the overhead of boxing the args.
 	IsTracingEnabled(ctx context.Context) bool
 }
 
diff --git a/sstable/reader.go b/sstable/reader.go
index a9994c93..5b21ff96 100644
--- a/sstable/reader.go
+++ b/sstable/reader.go
@@ -2850,20 +2850,14 @@ func (r *Reader) readBlock(
 
 	v := r.opts.Cache.Alloc(int(bh.Length + blockTrailerLen))
 	b := v.Buf()
-	readStartTime := time.Now()
+	readStopwatch := makeStopwatch()
 	var err error
 	if readHandle != nil {
 		_, err = readHandle.ReadAt(b, int64(bh.Offset))
 	} else {
 		_, err = r.readable.ReadAt(b, int64(bh.Offset))
 	}
-	readDuration := time.Since(readStartTime)
-	// TODO(sumeer): should the threshold be configurable.
-	const slowReadTracingThreshold = 5 * time.Millisecond
-	// The invariants.Enabled path is for deterministic testing.
-	if invariants.Enabled {
-		readDuration = slowReadTracingThreshold
-	}
+	readDuration := readStopwatch.stop()
 	// Call IsTracingEnabled to avoid the allocations of boxing integers into an
 	// interface{}, unless necessary.
 	if readDuration >= slowReadTracingThreshold && r.opts.LoggerAndTracer.IsTracingEnabled(ctx) {
@@ -3389,7 +3383,7 @@ func NewReader(f objstorage.Readable, o ReaderOptions, extraOpts ...ReaderOption
 		r.cacheID = r.opts.Cache.NewID()
 	}
 
-	footer, err := readFooter(f)
+	footer, err := readFooter(f, r.opts.LoggerAndTracer)
 	if err != nil {
 		r.err = err
 		return nil, r.Close()
@@ -3767,3 +3761,20 @@ func (s *simpleReadable) Size() int64 {
 func (s *simpleReadable) NewReadHandle() objstorage.ReadHandle {
 	return &s.rh
 }
+
+type deterministicStopwatchForTesting struct {
+	startTime time.Time
+}
+
+func makeStopwatch() deterministicStopwatchForTesting {
+	return deterministicStopwatchForTesting{startTime: time.Now()}
+}
+
+func (w deterministicStopwatchForTesting) stop() time.Duration {
+	dur := time.Since(w.startTime)
+	// The invariants.Enabled path is for deterministic testing.
+	if invariants.Enabled {
+		dur = slowReadTracingThreshold
+	}
+	return dur
+}
diff --git a/sstable/table.go b/sstable/table.go
index 3d966843..9819d3b6 100644
--- a/sstable/table.go
+++ b/sstable/table.go
@@ -68,8 +68,10 @@
 package sstable // import "github.com/cockroachdb/pebble/sstable"
 
 import (
+	"context"
 	"encoding/binary"
 	"io"
+	"time"
 
 	"github.com/cockroachdb/errors"
 	"github.com/cockroachdb/pebble/internal/base"
@@ -305,7 +307,10 @@ type footer struct {
 	footerBH    BlockHandle
 }
 
-func readFooter(f objstorage.Readable) (footer, error) {
+// TODO(sumeer): should the threshold be configurable.
+const slowReadTracingThreshold = 5 * time.Millisecond
+
+func readFooter(f objstorage.Readable, logger base.LoggerAndTracer) (footer, error) {
 	var footer footer
 	size := f.Size()
 	if size < minFooterLen {
@@ -317,11 +322,19 @@ func readFooter(f objstorage.Readable) (footer, error) {
 	if off < 0 {
 		off = 0
 	}
+	readStopwatch := makeStopwatch()
 	n, err := f.ReadAt(buf, off)
 	if err != nil && err != io.EOF {
 		return footer, errors.Wrap(err, "pebble/table: invalid table (could not read footer)")
 	}
 	buf = buf[:n]
+	readDuration := readStopwatch.stop()
+	// Call IsTracingEnabled to avoid the allocations of boxing integers into an
+	// interface{}, unless necessary.
+	if readDuration >= slowReadTracingThreshold && logger.IsTracingEnabled(context.TODO()) {
+		logger.Eventf(context.TODO(), "reading %d bytes took %s",
+			len(buf), readDuration.String())
+	}
 
 	switch magic := buf[len(buf)-len(rocksDBMagic):]; string(magic) {
 	case levelDBMagic:


Test patch:
diff --git a/db_test.go b/db_test.go
index e80f4b45..0dd822af 100644
--- a/db_test.go
+++ b/db_test.go
@@ -1337,7 +1337,7 @@ func TestTracing(t *testing.T) {
 	_, closer, err := d.Get([]byte("hello"))
 	require.NoError(t, err)
 	closer.Close()
-	readerInitTraceString := "reading 37 bytes took 5ms\nreading 628 bytes took 5ms\n"
+	readerInitTraceString := "reading 53 bytes took 5ms\nreading 37 bytes took 5ms\nreading 628 bytes took 5ms\n"
 	iterTraceString := "reading 27 bytes took 5ms\nreading 29 bytes took 5ms\n"
 	require.Equal(t, readerInitTraceString+iterTraceString, tracer.buf.String())
 
diff --git a/sstable/table_test.go b/sstable/table_test.go
index 3483238d..26082ba4 100644
--- a/sstable/table_test.go
+++ b/sstable/table_test.go
@@ -752,7 +752,7 @@ func TestFooterRoundTrip(t *testing.T) {
 							readable, err := NewSimpleReadable(f)
 							require.NoError(t, err)
 
-							result, err := readFooter(readable)
+							result, err := readFooter(readable, base.NoopLoggerAndTracer{})
 							require.NoError(t, err)
 							require.NoError(t, readable.Close())
 
@@ -805,7 +805,7 @@ func TestReadFooter(t *testing.T) {
 			readable, err := NewSimpleReadable(f)
 			require.NoError(t, err)
 
-			if _, err := readFooter(readable); err == nil {
+			if _, err := readFooter(readable, base.NoopLoggerAndTracer{}); err == nil {
 				t.Fatalf("expected %q, but found success", c.expected)
 			} else if !strings.Contains(err.Error(), c.expected) {
 				t.Fatalf("expected %q, but found %v", c.expected, err)