Repository: grafana/pyroscope
Pull request #4168: chore: add v2 integration tests
Author: login: korniltsev
Created: 2025-05-02T07:16:45Z | Merged: 2025-05-02T15:16:54Z

- modify integrations tests to run on both v1 and v2 configurations
- fix race in query frontend rng 
- fix race in segment writer shutdown 


Diff:
diff --git a/pkg/experiment/ingester/segment.go b/pkg/experiment/ingester/segment.go
index 355fcdcfd5..e048c3dec7 100644
--- a/pkg/experiment/ingester/segment.go
+++ b/pkg/experiment/ingester/segment.go
@@ -77,26 +77,34 @@ func (sh *shard) ingest(fn func(head segmentIngest)) segmentWaitFlushed {
 }
 
 func (sh *shard) loop(ctx context.Context) {
+	loopWG := new(sync.WaitGroup)
 	ticker := time.NewTicker(sh.sw.config.SegmentDuration)
-	defer ticker.Stop()
+	defer func() {
+		ticker.Stop()
+		// Blocking here to make sure no asynchronous code is executed on this shard once loop exits
+		// This is mostly needed to fix a race in our integration tests
+		loopWG.Wait()
+	}()
 	for {
 		select {
 		case <-ticker.C:
-			sh.flushSegment(context.Background())
+			sh.flushSegment(context.Background(), loopWG)
 		case <-ctx.Done():
-			sh.flushSegment(context.Background())
+			sh.flushSegment(context.Background(), loopWG)
 			return
 		}
 	}
 }
 
-func (sh *shard) flushSegment(ctx context.Context) {
+func (sh *shard) flushSegment(ctx context.Context, wg *sync.WaitGroup) {
 	sh.mu.Lock()
 	s := sh.segment
 	sh.segment = sh.sw.newSegment(sh, s.shard, sh.logger)
 	sh.mu.Unlock()
 
+	wg.Add(1)
 	go func() { // not blocking next ticks in case metastore/s3 latency is high
+		defer wg.Done()
 		t1 := time.Now()
 		s.inFlightProfiles.Wait()
 		s.debuginfo.waitInflight = time.Since(t1)
diff --git a/pkg/experiment/metastore/fsm/fsm.go b/pkg/experiment/metastore/fsm/fsm.go
index 02891e41c1..769994d65b 100644
--- a/pkg/experiment/metastore/fsm/fsm.go
+++ b/pkg/experiment/metastore/fsm/fsm.go
@@ -236,7 +236,7 @@ func (fsm *FSM) applyCommand(cmd *raft.Log) any {
 	if cmd.Index <= fsm.appliedIndex {
 		// Skip already applied commands at WAL restore.
 		// Note that the 0 index is a noop and is never applied to FSM.
-		return Response{}
+		return Response{} // todo this may result in nil deref if client does not exepect Response.Data to be nil , for example (svc *CompactionService) PollCompactionJobs
 	}
 
 	cmdType := strconv.FormatUint(uint64(e.Type), 10)
diff --git a/pkg/frontend/read_path/query_frontend/query_frontend.go b/pkg/frontend/read_path/query_frontend/query_frontend.go
index 0f0122508b..5ee08fdfd4 100644
--- a/pkg/frontend/read_path/query_frontend/query_frontend.go
+++ b/pkg/frontend/read_path/query_frontend/query_frontend.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"math/rand"
 	"slices"
+	"sync"
 
 	"github.com/go-kit/log"
 	"github.com/grafana/dskit/tenant"
@@ -53,6 +54,7 @@ func NewQueryFrontend(
 }
 
 var xrand = rand.New(rand.NewSource(4349676827832284783))
+var xrandMutex = sync.Mutex{} // todo fix the race properly
 
 func (q *QueryFrontend) Query(
 	ctx context.Context,
@@ -74,9 +76,11 @@ func (q *QueryFrontend) Query(
 		return new(queryv1.QueryResponse), nil
 	}
 	// Randomize the order of blocks to avoid hotspots.
+	xrandMutex.Lock()
 	xrand.Shuffle(len(blocks), func(i, j int) {
 		blocks[i], blocks[j] = blocks[j], blocks[i]
 	})
+	xrandMutex.Unlock()
 	// TODO(kolesnikovae): Should be dynamic.
 	p := queryplan.Build(blocks, 4, 20)
 
diff --git a/pkg/test/integration/helper.go b/pkg/test/integration/helper.go
index c3ffdea211..e355b7bf8e 100644
--- a/pkg/test/integration/helper.go
+++ b/pkg/test/integration/helper.go
@@ -45,6 +45,36 @@ import (
 	"github.com/grafana/pyroscope/pkg/util/connectgrpc"
 )
 
+func EachPyroscopeTest(t *testing.T, f func(p *PyroscopeTest, t *testing.T)) {
+	tests := []struct {
+		name string
+		f    func(t *testing.T) *PyroscopeTest
+	}{
+		{
+			"v1",
+			func(t *testing.T) *PyroscopeTest {
+				return new(PyroscopeTest).Configure(t, false)
+			},
+		},
+		{
+			"v2",
+			func(t *testing.T) *PyroscopeTest {
+				return new(PyroscopeTest).Configure(t, true)
+			},
+		},
+	}
+	for _, pt := range tests {
+		t.Run(pt.name, func(t *testing.T) {
+			p := pt.f(t)
+			p.start(t)
+			t.Cleanup(func() {
+				p.stop()
+			})
+			f(p, t)
+		})
+	}
+}
+
 type PyroscopeTest struct {
 	config         phlare.Config
 	it             *phlare.Phlare
@@ -53,22 +83,51 @@ type PyroscopeTest struct {
 	reg            *prometheus.Registry
 	httpPort       int
 	memberlistPort int
+	grpcPort       int
+	raftPort       int
 }
 
 const address = "127.0.0.1"
 const storeInMemory = "inmemory"
 
-func (p *PyroscopeTest) Start(t *testing.T) {
+func (p *PyroscopeTest) start(t *testing.T) {
+	var err error
+
+	p.it, err = phlare.New(p.config)
+
+	require.NoError(t, err)
+
+	p.wg.Add(1)
+	go func() {
+		defer p.wg.Done()
+		err := p.it.Run()
+		require.NoError(t, err)
+	}()
+	require.Eventually(t, func() bool {
+		return p.ringActive() && p.ready()
+	}, 30*time.Second, 100*time.Millisecond)
+}
 
-	ports, err := test.GetFreePorts(2)
+func (p *PyroscopeTest) Configure(t *testing.T, v2 bool) *PyroscopeTest {
+	ports, err := test.GetFreePorts(4)
 	require.NoError(t, err)
 	p.httpPort = ports[0]
 	p.memberlistPort = ports[1]
+	p.grpcPort = ports[2]
+	p.raftPort = ports[3]
+	t.Logf("ports: http %d memberlist %d grpc %d raft %d", p.httpPort, p.memberlistPort, p.grpcPort, p.raftPort)
 
 	p.prevReg = prometheus.DefaultRegisterer
 	p.reg = prometheus.NewRegistry()
 	prometheus.DefaultRegisterer = p.reg
 
+	if v2 {
+		err = os.Setenv("PYROSCOPE_V2_EXPERIMENT", "1")
+	} else {
+		err = os.Setenv("PYROSCOPE_V2_EXPERIMENT", "")
+	}
+	require.NoError(t, err)
+
 	err = cfg.DynamicUnmarshal(&p.config, []string{"pyroscope"}, flag.NewFlagSet("pyroscope", flag.ContinueOnError))
 	require.NoError(t, err)
 
@@ -76,6 +135,7 @@ func (p *PyroscopeTest) Start(t *testing.T) {
 	p.config.Server.HTTPListenAddress = address
 	p.config.Server.HTTPListenPort = p.httpPort
 	p.config.Server.GRPCListenAddress = address
+	p.config.Server.GRPCListenPort = p.grpcPort
 	p.config.Worker.SchedulerAddress = address
 	p.config.MemberlistKV.AdvertisePort = p.memberlistPort
 	p.config.MemberlistKV.TCPTransport.BindPort = p.memberlistPort
@@ -102,22 +162,29 @@ func (p *PyroscopeTest) Start(t *testing.T) {
 	p.config.LimitsConfig.MaxQueryLookback = 0
 	p.config.LimitsConfig.RejectOlderThan = 0
 	_ = p.config.Server.LogLevel.Set("debug")
-	p.it, err = phlare.New(p.config)
 
-	require.NoError(t, err)
-
-	p.wg.Add(1)
-	go func() {
-		defer p.wg.Done()
-		err := p.it.Run()
-		require.NoError(t, err)
-	}()
-	require.Eventually(t, func() bool {
-		return p.ringActive() && p.ready()
-	}, 30*time.Second, 100*time.Millisecond)
+	if v2 {
+		p.config.Storage.Bucket.Filesystem.Directory = t.TempDir()
+		p.config.Storage.Bucket.Backend = "filesystem"
+		p.config.LimitsConfig.WritePathOverrides.WritePath = "segment-writer"
+		p.config.LimitsConfig.ReadPathOverrides.EnableQueryBackend = true
+		p.config.SegmentWriter.LifecyclerConfig.MinReadyDuration = 0 * time.Second
+		p.config.SegmentWriter.LifecyclerConfig.Addr = address
+		p.config.SegmentWriter.MetadataUpdateTimeout = 0 * time.Second
+		p.config.Metastore.MinReadyDuration = 0 * time.Second
+		p.config.QueryBackend.Address = fmt.Sprintf("%s:%d", address, p.grpcPort)
+		p.config.Metastore.Address = fmt.Sprintf("%s:%d", address, p.grpcPort)
+		p.config.Metastore.Raft.ServerID = fmt.Sprintf("%s:%d", address, p.raftPort)
+		p.config.Metastore.Raft.BindAddress = fmt.Sprintf("%s:%d", address, p.raftPort)
+		p.config.Metastore.Raft.AdvertiseAddress = fmt.Sprintf("%s:%d", address, p.raftPort)
+		p.config.Metastore.Raft.Dir = t.TempDir()
+		p.config.Metastore.Raft.SnapshotsDir = t.TempDir()
+		p.config.Metastore.FSM.DataDir = t.TempDir()
+	}
+	return p
 }
 
-func (p *PyroscopeTest) Stop(t *testing.T) {
+func (p *PyroscopeTest) stop() {
 	defer func() {
 		prometheus.DefaultRegisterer = p.prevReg
 	}()
diff --git a/pkg/test/integration/ingest_jfr_test.go b/pkg/test/integration/ingest_jfr_test.go
index 5b68517c46..9c5e7a0d41 100644
--- a/pkg/test/integration/ingest_jfr_test.go
+++ b/pkg/test/integration/ingest_jfr_test.go
@@ -148,29 +148,27 @@ func TestNOJFRDump(t *testing.T) {
 }
 
 func TestIngestJFR(t *testing.T) {
-	p := new(PyroscopeTest)
-	p.Start(t)
-	defer p.Stop(t)
-
-	for _, testdatum := range jfrTestDatas {
-		t.Run(testdatum.jfr, func(t *testing.T) {
-
-			rb := p.NewRequestBuilder(t)
-			req := rb.IngestJFRRequestFiles(testdatum.jfr, testdatum.labels)
-			p.Ingest(t, req, testdatum.expectStatus)
-
-			if testdatum.expectStatus == 200 {
-				assert.NotEqual(t, len(testdatum.expectedMetrics), 0)
-				for _, metric := range testdatum.expectedMetrics {
-					goldFile := expectedPPROFFile(testdatum, metric)
-					t.Logf("%v gold file %s", metric, goldFile)
-					rb.Render(metric.name)
-					profile := rb.SelectMergeProfile(metric.name, metric.query)
-					verifyPPROF(t, profile, goldFile, metric)
+	EachPyroscopeTest(t, func(p *PyroscopeTest, t *testing.T) {
+		for _, testdatum := range jfrTestDatas {
+			t.Run(testdatum.jfr, func(t *testing.T) {
+
+				rb := p.NewRequestBuilder(t)
+				req := rb.IngestJFRRequestFiles(testdatum.jfr, testdatum.labels)
+				p.Ingest(t, req, testdatum.expectStatus)
+
+				if testdatum.expectStatus == 200 {
+					assert.NotEqual(t, len(testdatum.expectedMetrics), 0)
+					for _, metric := range testdatum.expectedMetrics {
+						goldFile := expectedPPROFFile(testdatum, metric)
+						t.Logf("%v gold file %s", metric, goldFile)
+						rb.Render(metric.name)
+						profile := rb.SelectMergeProfile(metric.name, metric.query)
+						verifyPPROF(t, profile, goldFile, metric)
+					}
 				}
-			}
-		})
-	}
+			})
+		}
+	})
 }
 
 func expectedPPROFFile(testdatum jfrTestData, metric expectedMetric) string {
@@ -185,18 +183,16 @@ func expectedPPROFFile(testdatum jfrTestData, metric expectedMetric) string {
 }
 
 func TestCorruptedJFR422(t *testing.T) {
-	p := new(PyroscopeTest)
-	p.Start(t)
-	defer p.Stop(t)
-
-	td := jfrTestDatas[0]
-	jfr, err := bench.ReadGzipFile(td.jfr)
-	require.NoError(t, err)
-	jfr[0] = 0 // corrupt jfr
+	EachPyroscopeTest(t, func(p *PyroscopeTest, t *testing.T) {
+		td := jfrTestDatas[0]
+		jfr, err := bench.ReadGzipFile(td.jfr)
+		require.NoError(t, err)
+		jfr[0] = 0 // corrupt jfr
 
-	rb := p.NewRequestBuilder(t)
-	req := rb.IngestJFRRequestBody(jfr, nil)
-	p.Ingest(t, req, 422)
+		rb := p.NewRequestBuilder(t)
+		req := rb.IngestJFRRequestBody(jfr, nil)
+		p.Ingest(t, req, 422)
+	})
 }
 
 func verifyPPROF(t *testing.T, resp *connect.Response[profilev1.Profile], expectedPPROF string, metric expectedMetric) {
diff --git a/pkg/test/integration/ingest_otlp_test.go b/pkg/test/integration/ingest_otlp_test.go
index 364fcc73f8..3232f0e6bf 100644
--- a/pkg/test/integration/ingest_otlp_test.go
+++ b/pkg/test/integration/ingest_otlp_test.go
@@ -91,63 +91,63 @@ var otlpTestDatas = []otlpTestData{
 func TestIngestOTLP(t *testing.T) {
 	for _, td := range otlpTestDatas {
 		t.Run(td.name, func(t *testing.T) {
-			p := new(PyroscopeTest)
-			p.Start(t)
-			defer p.Stop(t)
-			rb := p.NewRequestBuilder(t)
-			runNo := p.TempAppName()
-
-			profileBytes, err := os.ReadFile(td.profilePath)
-			require.NoError(t, err)
-			var profile = new(profilesv1.ExportProfilesServiceRequest)
-			err = proto.Unmarshal(profileBytes, profile)
-			require.NoError(t, err)
-
-			for _, rp := range profile.ResourceProfiles {
-				for _, sp := range rp.ScopeProfiles {
-					sp.Scope.Attributes = append(sp.Scope.Attributes, &commonv1.KeyValue{
-						Key: "test_run_no", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: runNo}},
-					})
-				}
-			}
+			EachPyroscopeTest(t, func(p *PyroscopeTest, t *testing.T) {
 
-			client := rb.OtelPushClient()
-			_, err = client.Export(context.Background(), profile)
-			require.NoError(t, err)
+				rb := p.NewRequestBuilder(t)
+				runNo := p.TempAppName()
 
-			for _, metric := range td.expectedProfiles {
+				profileBytes, err := os.ReadFile(td.profilePath)
+				require.NoError(t, err)
+				var profile = new(profilesv1.ExportProfilesServiceRequest)
+				err = proto.Unmarshal(profileBytes, profile)
+				require.NoError(t, err)
 
-				query := make(map[string]string)
-				for k, v := range metric.query {
-					query[k] = v
+				for _, rp := range profile.ResourceProfiles {
+					for _, sp := range rp.ScopeProfiles {
+						sp.Scope.Attributes = append(sp.Scope.Attributes, &commonv1.KeyValue{
+							Key: "test_run_no", Value: &commonv1.AnyValue{Value: &commonv1.AnyValue_StringValue{StringValue: runNo}},
+						})
+					}
 				}
-				query["test_run_no"] = runNo
 
-				resp := rb.SelectMergeProfile(metric.metricName, query)
+				client := rb.OtelPushClient()
+				_, err = client.Export(context.Background(), profile)
+				require.NoError(t, err)
 
-				assert.NotEmpty(t, resp.Msg.Sample)
-				assert.NotEmpty(t, resp.Msg.Function)
-				assert.NotEmpty(t, resp.Msg.Mapping)
-				assert.NotEmpty(t, resp.Msg.Location)
+				for _, metric := range td.expectedProfiles {
 
-				actualStr, err := strprofile.Stringify(resp.Msg, strprofile.Options{
-					NoTime:     true,
-					NoDuration: true,
-				})
-				assert.NoError(t, err)
+					query := make(map[string]string)
+					for k, v := range metric.query {
+						query[k] = v
+					}
+					query["test_run_no"] = runNo
 
-				pprofDumpFileName := strings.ReplaceAll(metric.expectedJsonPath, ".json", ".pprof.pb.bin") // for debugging
-				pprof, err := resp.Msg.MarshalVT()
-				assert.NoError(t, err)
-				err = os.WriteFile(pprofDumpFileName, pprof, 0644)
-				assert.NoError(t, err)
+					resp := rb.SelectMergeProfile(metric.metricName, query)
 
-				expectedBytes, err := os.ReadFile(metric.expectedJsonPath)
-				require.NoError(t, err)
+					assert.NotEmpty(t, resp.Msg.Sample)
+					assert.NotEmpty(t, resp.Msg.Function)
+					assert.NotEmpty(t, resp.Msg.Mapping)
+					assert.NotEmpty(t, resp.Msg.Location)
 
-				assert.Equal(t, string(expectedBytes), actualStr)
-			}
-			td.assertMetrics(t, p)
+					actualStr, err := strprofile.Stringify(resp.Msg, strprofile.Options{
+						NoTime:     true,
+						NoDuration: true,
+					})
+					assert.NoError(t, err)
+
+					pprofDumpFileName := strings.ReplaceAll(metric.expectedJsonPath, ".json", ".pprof.pb.bin") // for debugging
+					pprof, err := resp.Msg.MarshalVT()
+					assert.NoError(t, err)
+					err = os.WriteFile(pprofDumpFileName, pprof, 0644)
+					assert.NoError(t, err)
+
+					expectedBytes, err := os.ReadFile(metric.expectedJsonPath)
+					require.NoError(t, err)
+
+					assert.Equal(t, string(expectedBytes), actualStr)
+				}
+				td.assertMetrics(t, p)
+			})
 		})
 	}
 }
diff --git a/pkg/test/integration/ingest_pprof_test.go b/pkg/test/integration/ingest_pprof_test.go
index 16a43bc980..6db88d7ca3 100644
--- a/pkg/test/integration/ingest_pprof_test.go
+++ b/pkg/test/integration/ingest_pprof_test.go
@@ -248,202 +248,197 @@ var (
 )
 
 func TestIngest(t *testing.T) {
-	p := PyroscopeTest{}
-	p.Start(t)
-	defer p.Stop(t)
-
-	for _, td := range testdata {
-		t.Run(td.profile, func(t *testing.T) {
-			rb := p.NewRequestBuilder(t).
-				Spy(td.spyName)
-			req := rb.IngestPPROFRequest(td.profile, td.prevProfile, td.sampleTypeConfig)
-			p.Ingest(t, req, td.expectStatusIngest)
-
-			if td.expectStatusIngest == 200 {
-				for _, metric := range td.metrics {
-					rb.Render(metric.name)
-					profile := rb.SelectMergeProfile(metric.name, metric.query)
-					assertPPROF(t, profile, metric, td, td.fixAtIngest)
+	EachPyroscopeTest(t, func(p *PyroscopeTest, t *testing.T) {
+		for _, td := range testdata {
+			t.Run(td.profile, func(t *testing.T) {
+				rb := p.NewRequestBuilder(t).
+					Spy(td.spyName)
+				req := rb.IngestPPROFRequest(td.profile, td.prevProfile, td.sampleTypeConfig)
+				p.Ingest(t, req, td.expectStatusIngest)
+
+				if td.expectStatusIngest == 200 {
+					for _, metric := range td.metrics {
+						rb.Render(metric.name)
+						profile := rb.SelectMergeProfile(metric.name, metric.query)
+						assertPPROF(t, profile, metric, td, td.fixAtIngest)
+					}
 				}
-			}
-		})
-	}
+			})
+		}
+	})
 }
 
 func TestIngestPPROFFixPythonLinenumbers(t *testing.T) {
-	p := PyroscopeTest{}
-	p.Start(t)
-	defer p.Stop(t)
-	profile := pprof.RawFromProto(&profilev1.Profile{
-		SampleType: []*profilev1.ValueType{{
-			Type: 5,
-			Unit: 6,
-		}},
-		PeriodType: &profilev1.ValueType{
-			Type: 5, Unit: 6,
-		},
-		StringTable: []string{"", "main", "func1", "func2", "qwe.py", "cpu", "nanoseconds"},
-		Period:      1000000000,
-		Function: []*profilev1.Function{
-			{Id: 1, Name: 1, Filename: 4, SystemName: 1, StartLine: 239},
-			{Id: 2, Name: 2, Filename: 4, SystemName: 2, StartLine: 42},
-			{Id: 3, Name: 3, Filename: 4, SystemName: 3, StartLine: 7},
-		},
-		Location: []*profilev1.Location{
-			{Id: 1, Line: []*profilev1.Line{{FunctionId: 1, Line: 242}}},
-			{Id: 2, Line: []*profilev1.Line{{FunctionId: 2, Line: 50}}},
-			{Id: 3, Line: []*profilev1.Line{{FunctionId: 3, Line: 8}}},
-		},
-		Sample: []*profilev1.Sample{
-			{LocationId: []uint64{2, 1}, Value: []int64{10}},
-			{LocationId: []uint64{3, 1}, Value: []int64{13}},
-		},
-	})
+	EachPyroscopeTest(t, func(p *PyroscopeTest, t *testing.T) {
+
+		profile := pprof.RawFromProto(&profilev1.Profile{
+			SampleType: []*profilev1.ValueType{{
+				Type: 5,
+				Unit: 6,
+			}},
+			PeriodType: &profilev1.ValueType{
+				Type: 5, Unit: 6,
+			},
+			StringTable: []string{"", "main", "func1", "func2", "qwe.py", "cpu", "nanoseconds"},
+			Period:      1000000000,
+			Function: []*profilev1.Function{
+				{Id: 1, Name: 1, Filename: 4, SystemName: 1, StartLine: 239},
+				{Id: 2, Name: 2, Filename: 4, SystemName: 2, StartLine: 42},
+				{Id: 3, Name: 3, Filename: 4, SystemName: 3, StartLine: 7},
+			},
+			Location: []*profilev1.Location{
+				{Id: 1, Line: []*profilev1.Line{{FunctionId: 1, Line: 242}}},
+				{Id: 2, Line: []*profilev1.Line{{FunctionId: 2, Line: 50}}},
+				{Id: 3, Line: []*profilev1.Line{{FunctionId: 3, Line: 8}}},
+			},
+			Sample: []*profilev1.Sample{
+				{LocationId: []uint64{2, 1}, Value: []int64{10}},
+				{LocationId: []uint64{3, 1}, Value: []int64{13}},
+			},
+		})
 
-	tempProfileFile, err := os.CreateTemp("", "profile")
-	require.NoError(t, err)
-	_, err = profile.WriteTo(tempProfileFile)
-	assert.NoError(t, err)
-	tempProfileFile.Close()
-	defer os.Remove(tempProfileFile.Name())
-
-	rb := p.NewRequestBuilder(t).
-		Spy("pyspy")
-	req := rb.IngestPPROFRequest(tempProfileFile.Name(), "", "")
-	p.Ingest(t, req, 200)
-
-	renderedProfile := rb.SelectMergeProfile("process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil)
-	actual := bench.StackCollapseProto(renderedProfile.Msg, 0, 1)
-	expected := []string{
-		"qwe.py main;qwe.py func1 10",
-		"qwe.py main;qwe.py func2 13",
-	}
-	assert.Equal(t, expected, actual)
+		tempProfileFile, err := os.CreateTemp("", "profile")
+		require.NoError(t, err)
+		_, err = profile.WriteTo(tempProfileFile)
+		assert.NoError(t, err)
+		tempProfileFile.Close()
+		defer os.Remove(tempProfileFile.Name())
+
+		rb := p.NewRequestBuilder(t).
+			Spy("pyspy")
+		req := rb.IngestPPROFRequest(tempProfileFile.Name(), "", "")
+		p.Ingest(t, req, 200)
+
+		renderedProfile := rb.SelectMergeProfile("process_cpu:cpu:nanoseconds:cpu:nanoseconds", nil)
+		actual := bench.StackCollapseProto(renderedProfile.Msg, 0, 1)
+		expected := []string{
+			"qwe.py main;qwe.py func1 10",
+			"qwe.py main;qwe.py func2 13",
+		}
+		assert.Equal(t, expected, actual)
+	})
 }
 
 func TestGodeltaprofRelabelPush(t *testing.T) {
 	const blockSize = 1024
 	const metric = "godeltaprof_memory"
 
-	p := PyroscopeTest{}
-	p.Start(t)
-	defer p.Stop(t)
-
-	p1, _ := testhelper.NewProfileBuilder(time.Now().Add(-time.Second).UnixNano()).
-		MemoryProfile().
-		ForStacktraceString("my", "other").
-		AddSamples(239, 239*blockSize, 1000, 1000*blockSize).
-		Profile.MarshalVT()
-
-	p2, _ := testhelper.NewProfileBuilder(time.Now().UnixNano()).
-		MemoryProfile().
-		ForStacktraceString("my", "other").
-		AddSamples(3, 3*blockSize, 1000, 1000*blockSize).
-		Profile.MarshalVT()
-
-	rb := p.NewRequestBuilder(t)
-	rb.Push(rb.PushPPROFRequestFromBytes(p1, metric), 200, "")
-	rb.Push(rb.PushPPROFRequestFromBytes(p2, metric), 200, "")
-	renderedProfile := rb.SelectMergeProfile("memory:alloc_objects:count:space:bytes", nil)
-	actual := bench.StackCollapseProto(renderedProfile.Msg, 0, 1)
-	expected := []string{
-		"other;my 242",
-	}
-	assert.Equal(t, expected, actual)
+	EachPyroscopeTest(t, func(p *PyroscopeTest, t *testing.T) {
+
+		p1, _ := testhelper.NewProfileBuilder(time.Now().Add(-time.Second).UnixNano()).
+			MemoryProfile().
+			ForStacktraceString("my", "other").
+			AddSamples(239, 239*blockSize, 1000, 1000*blockSize).
+			Profile.MarshalVT()
+
+		p2, _ := testhelper.NewProfileBuilder(time.Now().UnixNano()).
+			MemoryProfile().
+			ForStacktraceString("my", "other").
+			AddSamples(3, 3*blockSize, 1000, 1000*blockSize).
+			Profile.MarshalVT()
+
+		rb := p.NewRequestBuilder(t)
+		rb.Push(rb.PushPPROFRequestFromBytes(p1, metric), 200, "")
+		rb.Push(rb.PushPPROFRequestFromBytes(p2, metric), 200, "")
+		renderedProfile := rb.SelectMergeProfile("memory:alloc_objects:count:space:bytes", nil)
+		actual := bench.StackCollapseProto(renderedProfile.Msg, 0, 1)
+		expected := []string{
+			"other;my 242",
+		}
+		assert.Equal(t, expected, actual)
+	})
 }
 
 func TestPushStringTableOOBSampleType(t *testing.T) {
 	const blockSize = 1024
 	const metric = "godeltaprof_memory"
 
-	p := PyroscopeTest{}
-	p.Start(t)
-	defer p.Stop(t)
-
-	testdata := []struct {
-		name        string
-		corrupt     func(p *testhelper.ProfileBuilder)
-		expectedErr string
-	}{
-		{
-			name: "sample type",
-			corrupt: func(p *testhelper.ProfileBuilder) {
-				p.SampleType[0].Type = 100500
+	EachPyroscopeTest(t, func(p *PyroscopeTest, t *testing.T) {
+
+		testdata := []struct {
+			name        string
+			corrupt     func(p *testhelper.ProfileBuilder)
+			expectedErr string
+		}{
+			{
+				name: "sample type",
+				corrupt: func(p *testhelper.ProfileBuilder) {
+					p.SampleType[0].Type = 100500
+				},
+				expectedErr: "sample type type string index out of range",
 			},
-			expectedErr: "sample type type string index out of range",
-		},
-		{
-			name: "function name",
-			corrupt: func(p *testhelper.ProfileBuilder) {
-				p.Function[0].Name = 100500
+			{
+				name: "function name",
+				corrupt: func(p *testhelper.ProfileBuilder) {
+					p.Function[0].Name = 100500
+				},
+				expectedErr: "function name string index out of range",
 			},
-			expectedErr: "function name string index out of range",
-		},
-		{
-			name: "mapping",
-			corrupt: func(p *testhelper.ProfileBuilder) {
-				p.Mapping[0].Filename = 100500
+			{
+				name: "mapping",
+				corrupt: func(p *testhelper.ProfileBuilder) {
+					p.Mapping[0].Filename = 100500
+				},
+				expectedErr: "mapping file name string index out of range",
 			},
-			expectedErr: "mapping file name string index out of range",
-		},
-		{
-			name: "Sample label",
-			corrupt: func(p *testhelper.ProfileBuilder) {
-				p.Sample[0].Label = []*profilev1.Label{{
-					Key: 100500,
-				}}
+			{
+				name: "Sample label",
+				corrupt: func(p *testhelper.ProfileBuilder) {
+					p.Sample[0].Label = []*profilev1.Label{{
+						Key: 100500,
+					}}
+				},
+				expectedErr: "sample label string index out of range",
 			},
-			expectedErr: "sample label string index out of range",
-		},
-		{
-			name: "String 0 not empty",
-			corrupt: func(p *testhelper.ProfileBuilder) {
-				p.StringTable[0] = "hmmm"
+			{
+				name: "String 0 not empty",
+				corrupt: func(p *testhelper.ProfileBuilder) {
+					p.StringTable[0] = "hmmm"
+				},
+				expectedErr: "string 0 should be empty string",
 			},
-			expectedErr: "string 0 should be empty string",
-		},
-	}
-	for _, td := range testdata {
-		t.Run(td.name, func(t *testing.T) {
-			p1 := testhelper.NewProfileBuilder(time.Now().Add(-time.Second).UnixNano()).
-				MemoryProfile().
-				ForStacktraceString("my", "other").
-				AddSamples(239, 239*blockSize, 1000, 1000*blockSize)
-			td.corrupt(p1)
-			p1bs, err := p1.Profile.MarshalVT()
-			require.NoError(t, err)
-
-			rb := p.NewRequestBuilder(t)
-			rb.Push(rb.PushPPROFRequestFromBytes(p1bs, metric), 400, td.expectedErr)
-		})
-	}
+		}
+		for _, td := range testdata {
+			t.Run(td.name, func(t *testing.T) {
+				p1 := testhelper.NewProfileBuilder(time.Now().Add(-time.Second).UnixNano()).
+					MemoryProfile().
+					ForStacktraceString("my", "other").
+					AddSamples(239, 239*blockSize, 1000, 1000*blockSize)
+				td.corrupt(p1)
+				p1bs, err := p1.Profile.MarshalVT()
+				require.NoError(t, err)
+
+				rb := p.NewRequestBuilder(t)
+				rb.Push(rb.PushPPROFRequestFromBytes(p1bs, metric), 400, td.expectedErr)
+			})
+		}
+	})
 }
 
 func TestPush(t *testing.T) {
-	p := new(PyroscopeTest)
-	p.Start(t)
-	defer p.Stop(t)
+	EachPyroscopeTest(t, func(p *PyroscopeTest, t *testing.T) {
 
-	for _, td := range testdata {
-		if td.prevProfile != "" {
-			continue
-		}
-		t.Run(td.profile, func(t *testing.T) {
-			rb := p.NewRequestBuilder(t)
+		for _, td := range testdata {
+			if td.prevProfile != "" {
+				continue
+			}
+			t.Run(td.profile, func(t *testing.T) {
+				rb := p.NewRequestBuilder(t)
 
-			req := rb.PushPPROFRequestFromFile(td.profile, td.metrics[0].name)
-			rb.Push(req, td.expectStatusPush, td.expectedError)
+				req := rb.PushPPROFRequestFromFile(td.profile, td.metrics[0].name)
+				rb.Push(req, td.expectStatusPush, td.expectedError)
 
-			if td.expectStatusPush == 200 {
-				for _, metric := range td.metrics {
-					rb.Render(metric.name)
-					profile := rb.SelectMergeProfile(metric.name, metric.query)
+				if td.expectStatusPush == 200 {
+					for _, metric := range td.metrics {
+						rb.Render(metric.name)
+						profile := rb.SelectMergeProfile(metric.name, metric.query)
 
-					assertPPROF(t, profile, metric, td, td.fixAtPush)
+						assertPPROF(t, profile, metric, td, td.fixAtPush)
+					}
 				}
-			}
-		})
-	}
+			})
+		}
+	})
 }
 
 func assertPPROF(t *testing.T, resp *connect.Response[profilev1.Profile], metric expectedMetric, testdatum pprofTestData, fix func(*pprof.Profile) *pprof.Profile) {


Reviews:
user: login: simonswine
body: LGTM

> https://github.com/grafana/pyroscope/pull/4168 change would not catch this.

Fully agreed, created another tracking issue
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-02T15:16:49Z
user: login: kolesnikovae
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-26T12:01:42Z
user: login: kolesnikovae
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-26T12:03:16Z
user: login: korniltsev
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-27T04:00:01Z
user: login: kolesnikovae
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-05-27T04:21:35Z

Review comments:
diff_hunk: @@ -236,7 +236,7 @@ func (fsm *FSM) applyCommand(cmd *raft.Log) any {
 	if cmd.Index <= fsm.appliedIndex {
 		// Skip already applied commands at WAL restore.
 		// Note that the 0 index is a noop and is never applied to FSM.
-		return Response{}
+		return Response{} // todo this may result in nil deref if client does not exepect Response.Data to be nil , for example (svc *CompactionService) PollCompactionJobs
path: pkg/experiment/metastore/fsm/fsm.go
user: login: kolesnikovae
body: I'm wondering how this could happen. Is there a way I can reproduce it?

This branch is taken when the WAL is restored – the response is not accessed as there's no caller on our end, therefore we don't care about its content.

The only scenario I can think of is when the FSM snapshot and the raft log are inconsistent: the state is from the future, and the log is from the past; thus, if the caller added a new raft command, it would satisfy `cmd.Index <= fsm.appliedIndex`, and the client would receive an empty response from FSM.

If this is the case, I'm not sure we want to "suppress" it. Instead, we should detect such situations and report a clear error. This can be done when we [propose a new raft command](https://github.com/grafana/pyroscope/blob/main/pkg/experiment/metastore/raftnode/node.go#L299) – all handlers return a response, the `nil` check here should be converted to an error. If the cluster entered such state, it should be forcibly recovered from a snapshot – the procedure we haven't implemented yet.

Note that a command cannot be applied to FSM _before_  it's committed to WAL. That would violate one of the core principles of Raft: State Machine Safety. If this is the case, things are _very_ bad.
created_at: 2025-05-26T12:01:41Z
author_association: COLLABORATOR
diff_hunk: @@ -53,6 +54,7 @@ func NewQueryFrontend(
 }
 
 var xrand = rand.New(rand.NewSource(4349676827832284783))
+var xrandMutex = sync.Mutex{} // todo fix the race properly
path: pkg/frontend/read_path/query_frontend/query_frontend.go
user: login: kolesnikovae
body: Thanks for fixing it! I think we could avoid the global random by sorting blocks based on the "random" part of the block UUID.
created_at: 2025-05-26T12:03:16Z
author_association: COLLABORATOR
diff_hunk: @@ -236,7 +236,7 @@ func (fsm *FSM) applyCommand(cmd *raft.Log) any {
 	if cmd.Index <= fsm.appliedIndex {
 		// Skip already applied commands at WAL restore.
 		// Note that the 0 index is a noop and is never applied to FSM.
-		return Response{}
+		return Response{} // todo this may result in nil deref if client does not exepect Response.Data to be nil , for example (svc *CompactionService) PollCompactionJobs
path: pkg/experiment/metastore/fsm/fsm.go
user: login: korniltsev
body: I hit a nil deref with a misconfigured pyroscope instance.
One of these was missing. 
```go
		p.config.Metastore.Raft.Dir = t.TempDir()
		p.config.Metastore.Raft.SnapshotsDir = t.TempDir()
		p.config.Metastore.FSM.DataDir = t.TempDir()
```
It does not happen in a correctly configured setup.
created_at: 2025-05-27T04:00:01Z
author_association: CONTRIBUTOR
diff_hunk: @@ -236,7 +236,7 @@ func (fsm *FSM) applyCommand(cmd *raft.Log) any {
 	if cmd.Index <= fsm.appliedIndex {
 		// Skip already applied commands at WAL restore.
 		// Note that the 0 index is a noop and is never applied to FSM.
-		return Response{}
+		return Response{} // todo this may result in nil deref if client does not exepect Response.Data to be nil , for example (svc *CompactionService) PollCompactionJobs
path: pkg/experiment/metastore/fsm/fsm.go
user: login: kolesnikovae
body: Phew! Thanks for the clarification – I'm still not very sure what exactly has happened, but I'll try to reproduce it
created_at: 2025-05-27T04:21:35Z
author_association: COLLABORATOR

PR comments:
user: login: simonswine
created_at: 2025-05-02T07:47:29Z
body: Plus point for also catching tenant isolation issues like #4160 
author_association: CONTRIBUTOR
user: login: korniltsev
created_at: 2025-05-02T11:40:37Z
body: > Plus point for also catching tenant isolation issues like https://github.com/grafana/pyroscope/pull/4160

#4168 change would not  catch this. We'd need to write some more test code. Totally agree we need this. At least the reproducer catching the bug. At most cover not only labelvalues, but the profiling data with a testcase. Ir should probably be a separate issue and separate PR.

author_association: CONTRIBUTOR