Repository: prometheus/prometheus (language: go)

Issue:
SIGSEGV after writing block
### What did you do?

After upgrading prometheus crashes after writing a block to disk:

```
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x7becc27ed8b3 pc=0x284122c]

goroutine 21496756 gp=0xc08c4bc380 m=39 mp=0xc093f06008 [running]:
runtime.throw({0x3942abb?, 0xc001eb3300?})
	/usr/local/go/src/runtime/panic.go:1023 +0x5c fp=0xc085183a58 sp=0xc085183a28 pc=0x43e95c
runtime.sigpanic()
	/usr/local/go/src/runtime/signal_unix.go:895 +0x285 fp=0xc085183ab8 sp=0xc085183a58 pc=0x457405
encoding/binary.Uvarint(...)
	/usr/local/go/src/encoding/binary/varint.go:72
github.com/prometheus/prometheus/tsdb/chunks.(*Reader).ChunkOrIterable(0xc00395bc70, {0x148f68b3, {0x0, 0x0}, 0x19076d5c700, 0x1907701b900, 0x0, 0x0, 0x0})
	/app/tsdb/chunks/chunks.go:681 +0x22c fp=0xc085183bb0 sp=0xc085183ab8 pc=0x284122c
github.com/prometheus/prometheus/tsdb.(*blockChunkReader).ChunkOrIterable(0x1?, {0x148f68b3, {0x0, 0x0}, 0x19076d5c700, 0x1907701b900, 0x0, 0x0, 0x0})
	<autogenerated>:1 +0x70 fp=0xc085183c08 sp=0xc085183bb0 pc=0x2b447f0
```

This always happens right after  it wrote a new block to disk:

log search for this crash it shows it always happens on the hour, i.e. when it writes the block

<img width="1556" alt="Screenshot 2024-07-05 at 10 54 36" src="https://github.com/prometheus/prometheus/assets/474727/ab4556ec-14e3-45b1-962c-abbf853c8395">

* Prometheus crashes after writing a block
* The block is valid when it starts up, and seems to contain the right amount of data
* `promtool tsdb validate` and `tsdb dump` seem to handle the block just fine
* only happens on a subset of our instances, only on the biggest (>1 million series)
```
    "headStats": {
      "numSeries": 1540181,
      "numLabelPairs": 58459,
      "chunkCount": 2742545,
      "minTime": 1720173600000,
      "maxTime": 1720177450840
    },
 ```
* crashes are not constant, some nodes crash more than others. Crash count in the last 48h, each cluster has 2 instances in ha pair:
```
 33 cluster-2
  10 cluster-4
   8 cluster-1
   6 cluster-3
```

### What did you expect to see?

Prometheus not to crash

### What did you see instead? Under which circumstances?



### System information

Linux 5.15.146+ x86_64

### Prometheus version

```text
prometheus, version 2.53.0 (branch: HEAD, revision: 4c35b9250afefede41c5f5acd76191f90f625898)
  build user:       root@7f8d89cbbd64
  build date:       20240619-07:39:12
  go version:       go1.22.4
  platform:         linux/amd64
  tags:             netgo,builtinassets,stringlabels
```


### Prometheus configuration file

_No response_

### Prometheus setup

This prometheus is managed by the prometheus operator in a (google) GKE k8s cluster. Disks are `pd-ssd`, standard persistent ssd storage offered by google.

The setup includes a thanos side car for shipping to a bucket and querying



### Alertmanager version

_No response_

### Alertmanager configuration file

_No response_

### Logs

```text
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x7becc27ed8b3 pc=0x284122c]

goroutine 21496756 gp=0xc08c4bc380 m=39 mp=0xc093f06008 [running]:
runtime.throw({0x3942abb?, 0xc001eb3300?})
	/usr/local/go/src/runtime/panic.go:1023 +0x5c fp=0xc085183a58 sp=0xc085183a28 pc=0x43e95c
runtime.sigpanic()
	/usr/local/go/src/runtime/signal_unix.go:895 +0x285 fp=0xc085183ab8 sp=0xc085183a58 pc=0x457405
encoding/binary.Uvarint(...)
	/usr/local/go/src/encoding/binary/varint.go:72
github.com/prometheus/prometheus/tsdb/chunks.(*Reader).ChunkOrIterable(0xc00395bc70, {0x148f68b3, {0x0, 0x0}, 0x19076d5c700, 0x1907701b900, 0x0, 0x0, 0x0})
	/app/tsdb/chunks/chunks.go:681 +0x22c fp=0xc085183bb0 sp=0xc085183ab8 pc=0x284122c
github.com/prometheus/prometheus/tsdb.(*blockChunkReader).ChunkOrIterable(0x1?, {0x148f68b3, {0x0, 0x0}, 0x19076d5c700, 0x1907701b900, 0x0, 0x0, 0x0})
	<autogenerated>:1 +0x70 fp=0xc085183c08 sp=0xc085183bb0 pc=0x2b447f0
github.com/prometheus/prometheus/tsdb.(*populateWithDelGenericSeriesIterator).next(0xc047264640, 0x1)
	/app/tsdb/querier.go:654 +0x36b fp=0xc085183d78 sp=0xc085183c08 pc=0x2b287eb
github.com/prometheus/prometheus/tsdb.(*populateWithDelChunkSeriesIterator).Next(0xc047264640)
	/app/tsdb/querier.go:825 +0x113 fp=0xc085183db8 sp=0xc085183d78 pc=0x2b293b3
github.com/prometheus/prometheus/storage.(*compactChunkIterator).Next(0xc0be977c20)
	/app/storage/merge.go:724 +0x82 fp=0xc085184070 sp=0xc085183db8 pc=0x2854782
github.com/prometheus/prometheus/storage/remote.StreamChunkedReadResponses({0x44251c0, 0xc13f854750}, 0x0, {0x445f720, 0xc0c36722f0}, {0xc14bd88000, 0x4, 0x4}, 0x100000, 0xc002078e70)
	/app/storage/remote/codec.go:246 +0x1bd fp=0xc085184338 sp=0xc085184070 pc=0x2a6211d
github.com/prometheus/prometheus/storage/remote.(*readHandler).remoteReadStreamedXORChunks.func1(0xc0851844e8, 0xc11db18d80, 0xc0025f0060, {0x445f058, 0xc11db18c60}, {0x7bed301a5278, 0xc09708de50}, {0x7bed301a52c8, 0xc09708de50}, 0xc0851844c8, ...)
	/app/storage/remote/read_handler.go:210 +0x211 fp=0xc085184460 sp=0xc085184338 pc=0x2a798b1
github.com/prometheus/prometheus/storage/remote.(*readHandler).remoteReadStreamedXORChunks(0xc0025f0060, {0x445f058, 0xc11db18c60}, {0x7bed301a5278, 0xc09708de50}, 0xc09708dea0, 0xc11db18d80, {0xc14bd88000, 0x4, 0x4})
	/app/storage/remote/read_handler.go:227 +0x245 fp=0xc085184530 sp=0xc085184460 pc=0x2a79525
github.com/prometheus/prometheus/storage/remote.(*readHandler).ServeHTTP(0xc0025f0060, {0x7bed301a5278, 0xc09708de50}, 0xc0cb24c6c0)
	/app/storage/remote/read_handler.go:110 +0x51a fp=0xc085184b10 sp=0xc085184530 pc=0x2a7845a
github.com/prometheus/prometheus/web/api/v1.(*API).remoteRead(0xc11db18d20?, {0x7bed301a5278?, 0xc09708de50?}, 0x4427a00?)
	/app/web/api/v1/api.go:1645 +0x32 fp=0xc085184b48 sp=0xc085184b10 pc=0x2b6eff2
github.com/prometheus/prometheus/web/api/v1.(*API).remoteRead-fm({0x7bed301a5278?, 0xc09708de50?}, 0xc11db18d20?)
	<autogenerated>:1 +0x36 fp=0xc085184b78 sp=0xc085184b48 pc=0x2b77eb6
github.com/prometheus/prometheus/web.(*Handler).testReady-fm.(*Handler).testReady.func1({0x7bed301a5278, 0xc09708de50}, 0xc0cb24c6c0)
	/app/web/web.go:548 +0x86 fp=0xc085184bc8 sp=0xc085184b78 pc=0x2b84166
net/http.HandlerFunc.ServeHTTP(0x7bed301a5278?, {0x7bed301a5278?, 0xc09708de50?}, 0x0?)
	/usr/local/go/src/net/http/server.go:2166 +0x29 fp=0xc085184bf0 sp=0xc085184bc8 pc=0x715d89
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerResponseSize.func1({0x7bed301a5278?, 0xc09708de00?}, 0xc0cb24c6c0)
	/go/pkg/mod/github.com/prometheus/client_golang@v1.19.1/prometheus/promhttp/instrument_server.go:296 +0xcf fp=0xc085184d10 sp=0xc085184bf0 pc=0x18d07ef
net/http.HandlerFunc.ServeHTTP(0xc11db18cc0?, {0x7bed301a5278?, 0xc09708de00?}, 0x50?)
	/usr/local/go/src/net/http/server.go:2166 +0x29 fp=0xc085184d38 sp=0xc085184d10 pc=0x715d89
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerDuration.func2({0x7bed301a5278, 0xc09708de00}, 0xc0cb24c6c0)
	/go/pkg/mod/github.com/prometheus/client_golang@v1.19.1/prometheus/promhttp/instrument_server.go:109 +0xc2 fp=0xc085184e60 sp=0xc085184d38 pc=0x18cf422
net/http.HandlerFunc.ServeHTTP(0x4455800?, {0x7bed301a5278?, 0xc09708de00?}, 0xc0931c0f28?)
	/usr/local/go/src/net/http/server.go:2166 +0x29 fp=0xc085184e88 sp=0xc085184e60 pc=0x715d89
github.com/prometheus/client_golang/prometheus/promhttp.InstrumentHandlerCounter.func1({0x4455800?, 0xc14493fc20?}, 0xc0cb24c6c0)
	/go/pkg/mod/github.com/prometheus/client_golang@v1.19.1/prometheus/promhttp/instrument_server.go:147 +0xc3 fp=0xc085184fa0 sp=0xc085184e88 pc=0x18d01a3
github.com/prometheus/prometheus/web.(*Handler).Run.setPathWithPrefix.func5.1({0x4455800, 0xc14493fc20}, 0xc0cb24c5a0)
	/app/web/web.go:808 +0xb3 fp=0xc085185000 sp=0xc085184fa0 pc=0x2b81b53
github.com/prometheus/common/route.(*Router).handle.func1({0x4455800, 0xc14493fc20}, 0xc0cb24c480, {0x0, 0x0, 0x41acf8?})
	/go/pkg/mod/github.com/prometheus/common@v0.54.0/route/route.go:83 +0x1e9 fp=0xc085185098 sp=0xc085185000 pc=0x2b58dc9
github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0xc0029b22a0, {0x4455800, 0xc14493fc20}, 0xc0cb24c480)
	/go/pkg/mod/github.com/julienschmidt/httprouter@v1.3.0/router.go:387 +0x7eb fp=0xc0851851a0 sp=0xc085185098 pc=0x2b55c4b
github.com/prometheus/common/route.(*Router).ServeHTTP(0x0?, {0x4455800?, 0xc14493fc20?}, 0x7?)
	/go/pkg/mod/github.com/prometheus/common@v0.54.0/route/route.go:126 +0x25 fp=0xc0851851d0 sp=0xc0851851a0 pc=0x2b593c5
github.com/prometheus/prometheus/web.(*Handler).Run.StripPrefix.func7({0x4455800, 0xc14493fc20}, 0xc0cb24c360)
	/usr/local/go/src/net/http/server.go:2209 +0x262 fp=0xc085185258 sp=0xc0851851d0 pc=0x2b81902
net/http.HandlerFunc.ServeHTTP(0xc0029b6090?, {0x4455800?, 0xc14493fc20?}, 0xc0931c12c0?)
	/usr/local/go/src/net/http/server.go:2166 +0x29 fp=0xc085185280 sp=0xc085185258 pc=0x715d89
net/http.(*ServeMux).ServeHTTP(0x445f058?, {0x4455800, 0xc14493fc20}, 0xc0cb24c360)
	/usr/local/go/src/net/http/server.go:2683 +0x1ad fp=0xc0851852d0 sp=0xc085185280 pc=0x717c6d
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.(*middleware).serveHTTP(0xc002d58000, {0x4449aa0, 0xc1449242a0}, 0xc0c310dd40, {0x4424340, 0xc0029b6000})
	/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.52.0/handler.go:212 +0x1003 fp=0xc085185888 sp=0xc0851852d0 pc=0x15bb4a3
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp.NewMiddleware.func1.1({0x4449aa0?, 0xc1449242a0?}, 0x7fff96775563?)
	/go/pkg/mod/go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp@v0.52.0/handler.go:73 +0x35 fp=0xc0851858c8 sp=0xc085185888 pc=0x15b9cf5
net/http.HandlerFunc.ServeHTTP(0x0?, {0x4449aa0?, 0xc1449242a0?}, 0xc144924380?)
	/usr/local/go/src/net/http/server.go:2166 +0x29 fp=0xc0851858f0 sp=0xc0851858c8 pc=0x715d89
github.com/prometheus/prometheus/web.(*Handler).Run.withStackTracer.func9({0x4449aa0?, 0xc1449242a0?}, 0xc144924380?)
	/app/web/web.go:104 +0x82 fp=0xc085185950 sp=0xc0851858f0 pc=0x2b81362
net/http.HandlerFunc.ServeHTTP(0x7fff96775563?, {0x4449aa0?, 0xc1449242a0?}, 0x6279c20?)
	/usr/local/go/src/net/http/server.go:2166 +0x29 fp=0xc085185978 sp=0xc085185950 pc=0x715d89
github.com/prometheus/exporter-toolkit/web.(*webHandler).ServeHTTP(0xc002d3aa00, {0x4449aa0, 0xc1449242a0}, 0xc0c310dd40)
	/go/pkg/mod/github.com/prometheus/exporter-toolkit@v0.11.0/web/handler.go:102 +0x41d fp=0xc085185b60 sp=0xc085185978 pc=0xa3e0dd
net/http.serverHandler.ServeHTTP({0x4435438?}, {0x4449aa0?, 0xc1449242a0?}, 0x6?)
	/usr/local/go/src/net/http/server.go:3137 +0x8e fp=0xc085185b90 sp=0xc085185b60 pc=0x71948e
net/http.(*conn).serve(0xc0a7e28ea0, {0x445f058, 0xc002e90660})
	/usr/local/go/src/net/http/server.go:2039 +0x5e8 fp=0xc085185fb8 sp=0xc085185b90 pc=0x714768
net/http.(*Server).Serve.gowrap3()
	/usr/local/go/src/net/http/server.go:3285 +0x28 fp=0xc085185fe0 sp=0xc085185fb8 pc=0x719ca8
runtime.goexit({})
	/usr/local/go/src/runtime/asm_amd64.s:1695 +0x1 fp=0xc085185fe8 sp=0xc085185fe0 pc=0x479701
created by net/http.(*Server).Serve in goroutine 426
	/usr/local/go/src/net/http/server.go:3285 +0x4b4
```



Discussion / hints:
all crashes in case the address is useful:

```
Jul 5, 2024 @ 13:00:46.389,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7c92458809d0 pc=0x284122c]
Jul 5, 2024 @ 11:00:52.365,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7c9c2d77579d pc=0x284122c]
Jul 5, 2024 @ 09:00:57.589,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7946a69a92cf pc=0x284122c]
Jul 5, 2024 @ 09:00:40.912,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7f0821065b61 pc=0x284122c]
Jul 5, 2024 @ 09:00:40.597,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7becc27ed8b3 pc=0x284122c]
Jul 5, 2024 @ 09:00:31.523,[signal SIGSEGV: segmentation violation code=0x1 addr=0x797bb947979a pc=0x284122c]
Jul 5, 2024 @ 05:00:41.974,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7c44ca1d0119 pc=0x284122c]
Jul 5, 2024 @ 05:00:35.309,[signal SIGSEGV: segmentation violation code=0x1 addr=0x78aa03a72897 pc=0x284122c]
Jul 5, 2024 @ 05:00:32.694,[signal SIGSEGV: segmentation violation code=0x1 addr=0x797a0ae0ace6 pc=0x284122c]
Jul 5, 2024 @ 03:00:40.729,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7e6705ce2cf5 pc=0x284122c]
Jul 5, 2024 @ 03:00:32.470,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7af476bb88f1 pc=0x284122c]
Jul 5, 2024 @ 01:00:42.368,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7fcf505e945f pc=0x284122c]
Jul 5, 2024 @ 01:00:40.742,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7fcf505cf895 pc=0x284122c]
Jul 4, 2024 @ 23:00:41.986,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7cbcf6775efc pc=0x284122c]
Jul 4, 2024 @ 21:00:34.998,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7975109dd0d0 pc=0x284122c]
Jul 4, 2024 @ 19:00:51.381,[signal SIGSEGV: segmentation violation code=0x1 addr=0x78ec17e67b0b pc=0x284122c]
Jul 4, 2024 @ 19:00:48.777,[signal SIGSEGV: segmentation violation code=0x1 addr=0x78ec17e6ad99 pc=0x284122c]
Jul 4, 2024 @ 19:00:36.356,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7dfddc0227b4 pc=0x284122c]
Jul 4, 2024 @ 17:00:46.642,[signal SIGSEGV: segmentation violation code=0x1 addr=0x792f5e483a6c pc=0x284122c]
Jul 4, 2024 @ 15:00:56.853,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7d28ddd3ba90 pc=0x284122c]
Jul 4, 2024 @ 15:00:56.231,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7d25c31bc708 pc=0x284122c]
Jul 4, 2024 @ 15:00:32.884,[signal SIGSEGV: segmentation violation code=0x1 addr=0x793ca916e070 pc=0x283d2f8]
Jul 4, 2024 @ 13:01:03.465,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7e43ab8a4c6d pc=0x284122c]
Jul 4, 2024 @ 13:01:03.257,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7870b91346cb pc=0x284122c]
Jul 4, 2024 @ 13:00:42.838,[signal SIGSEGV: segmentation violation code=0x1 addr=0x783c6cd1597c pc=0x284122c]
Jul 4, 2024 @ 13:00:40.823,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7812676ec3bb pc=0x284122c]
Jul 4, 2024 @ 13:00:40.815,[signal SIGSEGV: segmentation violation code=0x1 addr=0x78126771b301 pc=0x284122c]
Jul 4, 2024 @ 13:00:40.807,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7812673ede2a pc=0x284122c]
Jul 4, 2024 @ 13:00:39.320,[signal SIGSEGV: segmentation violation code=0x1 addr=0x781267717a62 pc=0x284122c]
Jul 4, 2024 @ 11:00:39.992,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7bfb0115f756 pc=0x284122c]
Jul 4, 2024 @ 09:00:44.834,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7be1c4f40453 pc=0x284122c]
Jul 4, 2024 @ 09:00:32.862,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7b1446308b91 pc=0x284122c]
Jul 4, 2024 @ 07:00:43.668,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7c0b45ea4294 pc=0x284122c]
Jul 4, 2024 @ 07:00:39.047,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7f7a148e13c8 pc=0x284122c]
Jul 4, 2024 @ 07:00:31.895,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7a6fac902a19 pc=0x284122c]
Jul 4, 2024 @ 05:01:11.435,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7975f2e9992b pc=0x284122c]
Jul 4, 2024 @ 05:00:34.029,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7903d9a3fc0f pc=0x284122c]
Jul 4, 2024 @ 03:00:38.932,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7bcccd80d5e1 pc=0x284122c]
Jul 4, 2024 @ 01:01:57.592,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7f7eb056aa8b pc=0x284122c]
Jul 4, 2024 @ 01:00:51.939,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7ca5906f089b pc=0x284122c]
Jul 4, 2024 @ 01:00:37.311,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7e16a1a161f6 pc=0x284122c]
Jul 3, 2024 @ 23:00:41.761,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7e8787e634bb pc=0x284122c]
Jul 3, 2024 @ 21:01:26.710,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7f4d2da08cda pc=0x284122c]
Jul 3, 2024 @ 21:01:01.142,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7b109be8139c pc=0x284122c]
Jul 3, 2024 @ 21:00:40.617,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7f4de9bf4a66 pc=0x284122c]
Jul 3, 2024 @ 21:00:31.424,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7949f326bb39 pc=0x284122c]
Jul 3, 2024 @ 21:00:30.169,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7949f326c18f pc=0x284122c]
Jul 3, 2024 @ 19:00:48.168,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7db2114dad9b pc=0x284122c]
Jul 3, 2024 @ 17:00:32.839,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7bbdd0aca6d4 pc=0x284122c]
Jul 3, 2024 @ 15:01:23.590,[signal SIGSEGV: segmentation violation code=0x1 addr=0x79bb58c59a3c pc=0x284122c]
Jul 3, 2024 @ 15:00:59.926,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7c949bfcfed9 pc=0x284122c]
Jul 3, 2024 @ 15:00:45.148,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7992485e192a pc=0x284122c]
Jul 3, 2024 @ 15:00:43.353,[signal SIGSEGV: segmentation violation code=0x1 addr=0x7992485faffe pc=0x284122c]
```
This one crashed in a different place; do you have the stack trace from it?
```
Jul 4, 2024 @ 15:00:32.884,[signal SIGSEGV: segmentation violation code=0x1 addr=0x793ca916e070 pc=0x283d2f8]
```
It looks like our logging setup broke the stacktrace down into individual lines and destroys the ordering between lines. I did my best to reconstruct ordering but if it seems wrong it probably is:

```
[signal SIGSEGV: segmentation violation code=0x1 addr=0x793ca916e070 pc=0x283d2f8]
goroutine 5542286 gp=0xc0493a1340 m=14 mp=0xc001480008 [running]:

runtime.throw({0x3942abb?, 0x8?})
	/usr/local/go/src/runtime/panic.go:1023 +0x5c fp=0xc01dea5788 sp=0xc01dea5758 pc=0x43e95c
runtime.sigpanic()
	/usr/local/go/src/runtime/signal_unix.go:895 +0x285 fp=0xc01dea57e8 sp=0xc01dea5788 pc=0x457405
github.com/dennwc/varint.Uvarint({0x793ca916e070?, 0xc01dea5828?, 0x45c72b?})
	/go/pkg/mod/github.com/dennwc/varint@v1.0.0/varint.go:75 +0x18 fp=0xc01dea57f0 sp=0xc01dea57e8 pc=0x283d2f8

github.com/prometheus/prometheus/tsdb/encoding.(*Decbuf).UvarintBytes(0xc01dea5868)

github.com/prometheus/prometheus/tsdb/encoding.(*Decbuf).Uvarint64(0xc01dea5868)
	/app/tsdb/encoding/encoding.go:243 +0x35 fp=0xc01dea5818 sp=0xc01dea57f0 pc=0x29f8a35

github.com/prometheus/prometheus/storage.(*chunkSeriesSetAdapter).Next(0xc062d40090?)
github.com/prometheus/prometheus/storage.(*genericMergeSeriesSet).Next(0xc086f38d80)
github.com/prometheus/prometheus/storage.(*genericMergeSeriesSet).Next(0xc086f39200)

github.com/prometheus/prometheus/storage/remote.(*readHandler).remoteReadStreamedXORChunks(0xc001b59860, {0x445f058, 0xc086f04d20}, {0x793cdabaf778, 0xc00dd493b0}, 0xc00dd49450, 0xc086f04db0, {0xc0638eed00, 0x4, 0x4})
github.com/prometheus/prometheus/storage/remote.(*readHandler).remoteReadStreamedXORChunks.func1(0xc01dea64e8, 0xc086f04db0, 0xc001b59860, {0x445f058, 0xc086f04d20}, {0x793cdabaf778, 0xc00dd493b0}, {0x793cdabaf7c8, 0xc00dd493b0}, 0xc01dea64c8, ...)

```
As its almost weekend we downgraded these instances back to 2.45. Monday we will revisit this and try to find the oldest version that has this issue
~This only happens in 2.53, tested a few versions before and none show.~. Wrong, was patient enough. Both 2.53 and 2.52 show it, 2.51 seems to be ok

I'll do a bisection to see if it can be pinned down to a commit, but it will take a while
@fbs do you have any news ?
Sort of. Some other things things came up but `6a66f1f57` also has the bug. Currently trying `2a2e2ed28` as a next bisection step
progress so far:

```
$ git bisect log
git bisect start
# status: waiting for both good and bad commits
# good: [c05c15512acb675e3f6cd662a6727854e93fc024] Cut v2.51.0 (#13787)
git bisect good c05c15512acb675e3f6cd662a6727854e93fc024
# status: waiting for bad commit, 1 good commit known
# bad: [879d80922a227c37df502e7315fad8ceb10a986d] Merge pull request #14063 from prometheus/prepare-2.52
git bisect bad 879d80922a227c37df502e7315fad8ceb10a986d
# bad: [6a66f1f5795fd40bd5a2022778b0788a8091b39d] Merge pull request #13859 from prometheus/beorn7/release
git bisect bad 6a66f1f5795fd40bd5a2022778b0788a8091b39d
# bad: [2a2e2ed28bc9f1f9205e874ed1be705e765ecb9b] chore(tsdb): set the wbl to nil as well in DBReadOnly.loadDataAsQueryable
git bisect bad 2a2e2ed28bc9f1f9205e874ed1be705e765ecb9b
# bad: [d1abc3f2557660728dc6a34eb2da1f32461b6665] Merge pull request #13777 from roidelapluie/remoteread2
git bisect bad d1abc3f2557660728dc6a34eb2da1f32461b6665
# good: [f1e9ec29f8f7d8c273e8980494be6aca18e1ef6f] Merge pull request #13752 from prometheus/superq/publish_docker_readme
git bisect good f1e9ec29f8f7d8c273e8980494be6aca18e1ef6f
```


Great work!  Taking the last two good/bad, 14 files changed: https://github.com/prometheus/prometheus/compare/f1e9ec29f8f7d8c273e8980494be6aca18e1ef6f..d1abc3f2557660728dc6a34eb2da1f32461b6665

Many of which are build files, comments, etc.

The only material change is from #13777, which aligns with remote-read being on your stack traces.
I theorise that closing the querier closes the memory-mapped file, which then makes it possible to get a memory fault.

@roidelapluie who authored that PR and @machine424 who reviewed it.
> This always happens right after it wrote a new block to disk:

#13777 is not related to writing, nor is writing implicated in the stack-traces, but a new block would be a reason for Linux to be changing memory mappings which could invalidate memory that otherwise contained useable data.

Fix patch:
diff --git a/storage/remote/read_handler.go b/storage/remote/read_handler.go
index 2a00ce897f8..ffc64c9c3fb 100644
--- a/storage/remote/read_handler.go
+++ b/storage/remote/read_handler.go
@@ -202,16 +202,34 @@ func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.Re
 				return err
 			}
 
-			chunks := h.getChunkSeriesSet(ctx, query, filteredMatchers)
-			if err := chunks.Err(); err != nil {
+			querier, err := h.queryable.ChunkQuerier(query.StartTimestampMs, query.EndTimestampMs)
+			if err != nil {
 				return err
 			}
+			defer func() {
+				if err := querier.Close(); err != nil {
+					level.Warn(h.logger).Log("msg", "Error on chunk querier close", "err", err.Error())
+				}
+			}()
+
+			var hints *storage.SelectHints
+			if query.Hints != nil {
+				hints = &storage.SelectHints{
+					Start:    query.Hints.StartMs,
+					End:      query.Hints.EndMs,
+					Step:     query.Hints.StepMs,
+					Func:     query.Hints.Func,
+					Grouping: query.Hints.Grouping,
+					Range:    query.Hints.RangeMs,
+					By:       query.Hints.By,
+				}
+			}
 
 			ws, err := StreamChunkedReadResponses(
 				NewChunkedWriter(w, f),
 				int64(i),
 				// The streaming API has to provide the series sorted.
-				chunks,
+				querier.Select(ctx, true, hints, filteredMatchers...),
 				sortedExternalLabels,
 				h.remoteReadMaxBytesInFrame,
 				h.marshalPool,
@@ -236,35 +254,6 @@ func (h *readHandler) remoteReadStreamedXORChunks(ctx context.Context, w http.Re
 	}
 }
 
-// getChunkSeriesSet executes a query to retrieve a ChunkSeriesSet,
-// encapsulating the operation in its own function to ensure timely release of
-// the querier resources.
-func (h *readHandler) getChunkSeriesSet(ctx context.Context, query *prompb.Query, filteredMatchers []*labels.Matcher) storage.ChunkSeriesSet {
-	querier, err := h.queryable.ChunkQuerier(query.StartTimestampMs, query.EndTimestampMs)
-	if err != nil {
-		return storage.ErrChunkSeriesSet(err)
-	}
-	defer func() {
-		if err := querier.Close(); err != nil {
-			level.Warn(h.logger).Log("msg", "Error on chunk querier close", "err", err.Error())
-		}
-	}()
-
-	var hints *storage.SelectHints
-	if query.Hints != nil {
-		hints = &storage.SelectHints{
-			Start:    query.Hints.StartMs,
-			End:      query.Hints.EndMs,
-			Step:     query.Hints.StepMs,
-			Func:     query.Hints.Func,
-			Grouping: query.Hints.Grouping,
-			Range:    query.Hints.RangeMs,
-			By:       query.Hints.By,
-		}
-	}
-	return querier.Select(ctx, true, hints, filteredMatchers...)
-}
-
 // filterExtLabelsFromMatchers change equality matchers which match external labels
 // to a matcher that looks for an empty label,
 // as that label should not be present in the storage.


Test patch: