Repository: thanos-io/thanos (Go)

Issue:
Store The request returns inconsistent data and reports an error
store error 

**Thanos, Prometheus and Golang version used**:  prometheus 2.17 

**Object Storage Provider**: ceph S3

**What happened**: query give data error

this is same grafana panel !

![image](https://user-images.githubusercontent.com/48743664/130465438-a5f94647-9e00-4da8-b761-c1407b97fd04.png)

I set the address of the query on grafana 

![image](https://user-images.githubusercontent.com/48743664/130465087-43e9c75a-d658-41cd-9d1f-30834e889920.png)

when i click this button like this , it not same

![image](https://user-images.githubusercontent.com/48743664/130465612-fd170467-c7fb-4886-9bf0-6125d5941d0f.png)

I think store have some error

**What you expected to happen**: 

I want to get all 

**How to reproduce it (as minimally and precisely as possible)**:

I set 3 receives then I set  

```
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: thanos-receive
spec:
  serviceName: thanos-receive
  replicas: 3
  selector:
    matchLabels:
      app: thanos-receive
  template:
    metadata:
      labels:
        app: thanos-receive
    spec:
      containers:
      - name: thanos-receive
        image:  thanosio/thanos:main-2021-08-18-5cfa3ff6
        imagePullPolicy: IfNotPresent
        command:
        - /bin/sh
        - "-c"
        - |
          PRE_HOSTNAME_NAME=`hostname | grep -E -o "[a-z-]+[a-z]"`
          ALL_ENDPOINTS_PEER_ADDRESS=""

          i=0
          while [ "$i" -le $(($NODE_NUMBER-1)) ]; do
            ALL_ENDPOINTS_PEER_ADDRESS=$ALL_ENDPOINTS_PEER_ADDRESS\"${PRE_HOSTNAME_NAME}-$i.${STS_SERVICENAME}:10901\",
            i=$(( i + 1 ))
          done

          ALL_ENDPOINTS_PEER_ADDRESS=${ALL_ENDPOINTS_PEER_ADDRESS:0:-1}

          echo ${ALL_ENDPOINTS_PEER_ADDRESS}


          mkdir /runningConfig/
          cp /baseConfig/storageConfig.yaml /runningConfig/storageConfig.yaml
          cp /baseConfig/hashrings-file /runningConfig/hashrings-file


          sed -i 's#${BUCKET_NAME}#'${BUCKET_NAME}'#g' /runningConfig/storageConfig.yaml
          sed -i 's#${CPHEGW_ADDRESS}#'${CPHEGW_ADDRESS}'#g' /runningConfig/storageConfig.yaml
          sed -i 's#${ACCESS_KEY}#'${ACCESS_KEY}'#g' /runningConfig/storageConfig.yaml
          sed -i 's#${SECRET_KEY}#'${SECRET_KEY}'#g' /runningConfig/storageConfig.yaml

          sed -i 's#${ALL_ENDPOINTS_PEER_ADDRESS}#'${ALL_ENDPOINTS_PEER_ADDRESS}'#g' /runningConfig/hashrings-file

          thanos receive \
            --log.level=debug \
            --grpc-address=${PEER_ADDRESS} \
            --http-address=${EXPORTER_ADDRESS} \
            --remote-write.address=${CONST_REMOTE_WRITE_ADDRESS} \
            --objstore.config-file=/runningConfig/storageConfig.yaml \
            --tsdb.path=/var/thanos/receive \
            --tsdb.retention=${RETENTION_TIME} \
            --label=${MERGE_LABEL1} \
            --receive.replication-factor=${NODE_NUMBER} \
            --receive.hashrings-file=/runningConfig/hashrings-file \
            --receive.local-endpoint=${HOSTNAME}.${STS_SERVICENAME}:10901
            
        env:
        - name: PEER_ADDRESS
          value: "0.0.0.0:10901"
        - name: EXPORTER_ADDRESS
          value: "0.0.0.0:10902"
        - name: CONST_REMOTE_WRITE_ADDRESS
          value: "0.0.0.0:19291"
        - name: RETENTION_TIME
          value: "1d"
        - name: MERGE_LABEL1
          value: 'receive_cluster="it_k8s_cluster"'
        - name: BUCKET_NAME
          value: thanos_data
        - name: CPHEGW_ADDRESS
          value: cephgw
        - name: ACCESS_KEY
          value: 64PN7CM714RWT4QZ4ZY8
        - name: SECRET_KEY
          value: LunQhGePzrzdnlS0b5EqAGRaFBwBbw7OKFWrdKjS
        - name: NODE_NUMBER
          value: "3"
        - name: STS_SERVICENAME
          value: thanos-receive
        volumeMounts:
        - name: configs
          mountPath: /baseConfig/
        livenessProbe:
          tcpSocket:
            port: 10902
          periodSeconds: 15
          timeoutSeconds: 15
          failureThreshold: 3
        readinessProbe:
          tcpSocket:
            port: 10902
          periodSeconds: 15
          timeoutSeconds: 15
          failureThreshold: 3
      volumes:
      - name: configs
        configMap:
          name: thanos-receive
          items:
          - key: storageConfig.yaml
            path: storageConfig.yaml
          - key: hashrings-file
            path: hashrings-file
```

```
apiVersion: v1
kind: ConfigMap
metadata:
  name: thanos-receive
  namespace: default
  labels:
    app: thanos-receive
data:
  storageConfig.yaml: |
    type: S3
    config:
      bucket: ${BUCKET_NAME}
      endpoint: ${CPHEGW_ADDRESS}
      access_key: ${ACCESS_KEY}
      secret_key: ${SECRET_KEY}

  hashrings-file: |
    [
      {
        "endpoints": [
            ${ALL_ENDPOINTS_PEER_ADDRESS}
        ]
      }
    ]
```
```
apiVersion: v1
kind: Service
metadata:
  name: thanos-receive-sts
  namespace: default
spec:
  clusterIP: None
  selector:
    app: thanos-receive
```
```
apiVersion: v1
kind: Service
metadata:
  name: thanos-receive
  namespace: default
spec:
  type: ClusterIP
  selector:
    app: thanos-receive
  ports:
  - name: thanos-receive-port
    port: 1929
    targetPort: 19291
  - name: thanos-receive-grpc
    port: 10901
    targetPort: 10901
```
store config 
```
apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: thanos-store
  namespace: default
spec:
  serviceName: thanos-store-sts
  replicas: 1
  selector:
    matchLabels:
      app: thanos-store
  template:
    metadata:
      labels:
        app: thanos-store
    spec:
      containers:
      - name: thanos-store
        image: thanosio/thanos:main-2021-08-18-5cfa3ff6
        command:
        - /bin/sh
        - "-c"
        - |
          mkdir /runningConfig/
          cp /baseConfig/storageConfig.yaml /runningConfig/storageConfig.yaml

          sed -i 's#${BUCKET_NAME}#'${BUCKET_NAME}'#g' /runningConfig/storageConfig.yaml
          sed -i 's#${CPHEGW_ADDRESS}#'${CPHEGW_ADDRESS}'#g' /runningConfig/storageConfig.yaml
          sed -i 's#${ACCESS_KEY}#'${ACCESS_KEY}'#g' /runningConfig/storageConfig.yaml
          sed -i 's#${SECRET_KEY}#'${SECRET_KEY}'#g' /runningConfig/storageConfig.yaml

          thanos store \
            --log.level=debug \
            --data-dir=/cache-data/ \
            --objstore.config-file=/runningConfig/storageConfig.yaml
        env:
        - name: BUCKET_NAME
          value: thanos_data
        - name: CPHEGW_ADDRESS
          value: cephgw
        - name: ACCESS_KEY
          value: 64PN7CM714RWT4QZ4ZY8
        - name: SECRET_KEY
          value: LunQhGePzrzdnlS0b5EqAGRaFBwBbw7OKFWrdKjS
        volumeMounts:
        - name: cache-data
          mountPath: /cache-data/
        - name: configs
          mountPath: /baseConfig/
      volumes:
      - name: configs
        configMap:
          name: thanos-receive
          items:
          - key: storageConfig.yaml
            path: storageConfig.yaml
  volumeClaimTemplates:
  - metadata:
      name: cache-data
    spec:
      accessModes: ["ReadWriteOnce"]
      storageClassName: cephrbd
      resources:
        requests:
          storage: 50G
```
Query
```
apiVersion: apps/v1
kind: Deployment
metadata:
  name: thanos-query
  namespace: default
spec:
  replicas: 1
  selector:
    matchLabels:
      app: thanos-query
  template:
    metadata:
      labels:
        app: thanos-query
    spec:
      containers:
      - name: thanos-query
        image: thanosio/thanos:main-2021-08-18-5cfa3ff6
        command:
        - /bin/sh
        - "-c"
        - |
          thanos query \
          --log.level=debug \
          --grpc-address=0.0.0.0:10901 \
          --http-address=0.0.0.0:9090 \
          --query.replica-label=receive_cluster --store=thanos-store:10901
          # Debugging Comments # --store=10.120.90.51:10901 \
          # Debugging Comments # --store=10.120.61.187:10901 \
          # Debugging Comments # --store=10.120.95.123:10901 \
          # Debugging Comments # --store=thanos-store:10901
```

**logs**
```
kubectl logs thanos-store-0
```
```
........
level=debug ts=2021-08-23T14:21:27.923501074Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:2 postingsTouchedSizeSum:595 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:2 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:10009 seriesTouched:1 seriesTouchedSizeSum:37 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:820 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:11613684 getAllDuration:11947506 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:44263}" err=null
level=debug ts=2021-08-23T14:21:27.923903688Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:2 postingsTouchedSizeSum:595 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:2 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:15013 seriesTouched:1 seriesTouchedSizeSum:37 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:118 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:12187535 getAllDuration:12695082 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:43743}" err=null
level=debug ts=2021-08-23T14:21:27.924661217Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:2 postingsTouchedSizeSum:595 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:2 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:29570 seriesTouched:1 seriesTouchedSizeSum:37 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:729 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:5326084 getAllDuration:5856086 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:29386}" err=null
level=debug ts=2021-08-23T14:21:27.968879788Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:2 postingsTouchedSizeSum:595 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:2 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:15982 seriesTouched:1 seriesTouchedSizeSum:37 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:364 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:5408643 getAllDuration:5886722 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:70799}" err=null
level=debug ts=2021-08-23T14:21:27.969447836Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:2 postingsTouchedSizeSum:595 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:2 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:12163 seriesTouched:1 seriesTouchedSizeSum:37 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:359 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:5821524 getAllDuration:6499857 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:27863}" err=null
level=debug ts=2021-08-23T14:21:27.969795098Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:2 postingsTouchedSizeSum:595 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:2 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:11343 seriesTouched:1 seriesTouchedSizeSum:37 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:225 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:6351576 getAllDuration:6760840 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:53982}" err=null
level=debug ts=2021-08-23T14:21:27.971586112Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:2 postingsTouchedSizeSum:595 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:2 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:38917 seriesTouched:1 seriesTouchedSizeSum:37 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:118 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:8005692 getAllDuration:8495285 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:42492}" err=null
level=debug ts=2021-08-23T14:21:28.053841031Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:2 postingsTouchedSizeSum:595 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:2 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:10286 seriesTouched:1 seriesTouchedSizeSum:37 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:359 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:9041294 getAllDuration:9411383 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:36749}" err=null
level=debug ts=2021-08-23T14:21:28.054136444Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:146 postingsTouchedSizeSum:3047 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:146 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:108861 seriesTouched:2 seriesTouchedSizeSum:94 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:2 chunksTouchedSizeSum:349 chunksFetched:2 chunksFetchedSizeSum:26576 chunksFetchCount:1 chunksFetchDurationSum:4989136 getAllDuration:8285301 mergedSeriesCount:2 mergedChunksCount:2 mergeDuration:66525}" err=null
level=debug ts=2021-08-23T14:21:28.056119146Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:146 postingsTouchedSizeSum:3047 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:146 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:322637 seriesTouched:2 seriesTouchedSizeSum:94 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:2 chunksTouchedSizeSum:236 chunksFetched:2 chunksFetchedSizeSum:21773 chunksFetchCount:1 chunksFetchDurationSum:3787636 getAllDuration:9920295 mergedSeriesCount:2 mergedChunksCount:2 mergeDuration:81515}" err=null
level=debug ts=2021-08-23T14:21:28.08841768Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:146 postingsTouchedSizeSum:3047 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:146 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:3573996 seriesTouched:2 seriesTouchedSizeSum:94 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:2 chunksTouchedSizeSum:236 chunksFetched:2 chunksFetchedSizeSum:21773 chunksFetchCount:1 chunksFetchDurationSum:5441849 getAllDuration:13330566 mergedSeriesCount:2 mergedChunksCount:2 mergeDuration:64439}" err=null
level=debug ts=2021-08-23T14:21:28.115353198Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:146 postingsTouchedSizeSum:3047 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:146 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:107282 seriesTouched:2 seriesTouchedSizeSum:94 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:2 chunksTouchedSizeSum:349 chunksFetched:2 chunksFetchedSizeSum:26583 chunksFetchCount:1 chunksFetchDurationSum:6468327 getAllDuration:9950961 mergedSeriesCount:2 mergedChunksCount:2 mergeDuration:58099}" err=null
level=debug ts=2021-08-23T14:21:28.118852414Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:146 postingsTouchedSizeSum:3047 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:146 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:102425 seriesTouched:2 seriesTouchedSizeSum:94 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:2 chunksTouchedSizeSum:236 chunksFetched:2 chunksFetchedSizeSum:21773 chunksFetchCount:1 chunksFetchDurationSum:8128333 getAllDuration:12524197 mergedSeriesCount:2 mergedChunksCount:2 mergeDuration:44126}" err=null
level=debug ts=2021-08-23T14:21:28.120633289Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:3 postingsTouchedSizeSum:747 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:3 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:17639 seriesTouched:1 seriesTouchedSizeSum:41 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:571 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:6257891 getAllDuration:6839276 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:43666}" err=null
level=debug ts=2021-08-23T14:21:28.155537749Z caller=bucket.go:1078 msg="stats query processed" stats="&{blocksQueried:1 postingsTouched:3 postingsTouchedSizeSum:747 postingsToFetch:0 postingsFetched:0 postingsFetchedSizeSum:0 postingsFetchCount:0 postingsFetchDurationSum:0 cachedPostingsCompressions:0 cachedPostingsCompressionErrors:0 cachedPostingsOriginalSizeSum:0 cachedPostingsCompressedSizeSum:0 cachedPostingsCompressionTimeSum:0 cachedPostingsDecompressions:3 cachedPostingsDecompressionErrors:0 cachedPostingsDecompressionTimeSum:32754 seriesTouched:1 seriesTouchedSizeSum:41 seriesFetched:0 seriesFetchedSizeSum:0 seriesFetchCount:0 seriesFetchDurationSum:0 chunksTouched:1 chunksTouchedSizeSum:577 chunksFetched:1 chunksFetchedSizeSum:16000 chunksFetchCount:1 chunksFetchDurationSum:4388240 getAllDuration:5168189 mergedSeriesCount:1 mergedChunksCount:1 mergeDuration:0}" err=null
level=error ts=2021-08-23T14:21:28.156452306Z caller=grpc.go:70 service=gRPC/server component=store msg="recovered from panic" panic="runtime error: slice bounds out of range [:751] with capacity 605" stack="goroutine 4558 [running]:\nruntime/debug.Stack(0x202b1e0, 0xc000987fb0, 0xc000758740)\n\t/usr/local/go/src/runtime/debug/stack.go:24 +0x9f\ngithub.com/thanos-io/thanos/pkg/server/grpc.New.func1(0x1c09740, 0xc0006a62b8, 0x1, 0xc0007587d0)\n\t/home/circleci/project/pkg/server/grpc/grpc.go:70 +0xf2\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery.WithRecoveryHandler.func1.1(0x205d470, 0xc000f8e8d0, 0x1c09740, 0xc0006a62b8, 0xc00045b080, 0x1ddaf50)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/recovery/options.go:33 +0x39\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery.recoverFrom(0x205d470, 0xc000f8e8d0, 0x1c09740, 0xc0006a62b8, 0xc000448680, 0x0, 0xc)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/recovery/interceptors.go:53 +0x5a\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery.StreamServerInterceptor.func1.1(0x206d138, 0xc0006100c0, 0xc000102dd0, 0xc000759c00)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/recovery/interceptors.go:41 +0xa5\npanic(0x1c09740, 0xc0006a62b8)\n\t/usr/local/go/src/runtime/panic.go:965 +0x1b9\ngithub.com/thanos-io/thanos/pkg/store/storepb.(*SeriesResponse).Marshal(0xc000b0d0c0, 0x1cd1c00, 0xc000b0d0c0, 0x7f0e83934738, 0xc000b0d0c0, 0xc00052e801)\n\t/home/circleci/project/pkg/store/storepb/custom.go:526 +0x385\ngoogle.golang.org/grpc/encoding/proto.codec.Marshal(0x1cd1c00, 0xc000b0d0c0, 0x7f0eaa5ce108, 0x10, 0xc0009465e0, 0x100000000000003, 0xc00006ec00)\n\t/home/circleci/go/pkg/mod/google.golang.org/grpc@v1.29.1/encoding/proto/proto.go:70 +0x199\ngoogle.golang.org/grpc.encode(0x7f0e83932d40, 0x2e50080, 0x1cd1c00, 0xc000b0d0c0, 0x2, 0xc0009465e8, 0x2, 0x40db9b, 0x0)\n\t/home/circleci/go/pkg/mod/google.golang.org/grpc@v1.29.1/rpc_util.go:545 +0x52\ngoogle.golang.org/grpc.prepareMsg(0x1cd1c00, 0xc000b0d0c0, 0x7f0e83932d40, 0x2e50080, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc00068ed08, ...)\n\t/home/circleci/go/pkg/mod/google.golang.org/grpc@v1.29.1/stream.go:1518 +0x10f\ngoogle.golang.org/grpc.(*serverStream).SendMsg(0xc0006100c0, 0x1cd1c00, 0xc000b0d0c0, 0x0, 0x0)\n\t/home/circleci/go/pkg/mod/google.golang.org/grpc@v1.29.1/stream.go:1411 +0xfb\ngithub.com/grpc-ecosystem/go-grpc-prometheus.(*monitoredServerStream).SendMsg(0xc0005888d0, 0x1cd1c00, 0xc000b0d0c0, 0x7f0eaa5ce5b8, 0x30)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:156 +0x4b\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors.(*monitoredServerStream).SendMsg(0xc000f8e990, 0x1cd1c00, 0xc000b0d0c0, 0x943027c, 0xbcf9b4e63746)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/server.go:64 +0x6d\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors.(*monitoredServerStream).SendMsg(0xc000f8ea20, 0x1cd1c00, 0xc000b0d0c0, 0x7f0e836b5600, 0x20)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/server.go:64 +0x6d\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors.(*monitoredServerStream).SendMsg(0xc000f8ea80, 0x1cd1c00, 0xc000b0d0c0, 0x4082a80000000001, 0xc000b0d0c0)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/server.go:64 +0x6d\ngithub.com/thanos-io/thanos/pkg/store/storepb.(*storeSeriesServer).Send(0xc0009d66a0, 0xc000b0d0c0, 0x1, 0x255)\n\t/home/circleci/project/pkg/store/storepb/rpc.pb.go:869 +0x49\ngithub.com/thanos-io/thanos/pkg/store.(*BucketStore).Series.func6(0x205d470, 0xc000c0b110)\n\t/home/circleci/project/pkg/store/bucket.go:1139 +0x23d\ngithub.com/thanos-io/thanos/pkg/tracing.DoInSpan(0x205d470, 0xc000f8e9f0, 0x1d52b6d, 0x16, 0xc00068f508, 0x0, 0x0, 0x0)\n\t/home/circleci/project/pkg/tracing/tracing.go:72 +0xcf\ngithub.com/thanos-io/thanos/pkg/store.(*BucketStore).Series(0xc00053a420, 0xc000905440, 0x2070f98, 0xc0009d66a0, 0x0, 0x0)\n\t/home/circleci/project/pkg/store/bucket.go:1109 +0xff4\ngithub.com/thanos-io/thanos/pkg/store/storepb._Store_Series_Handler(0x1c41d20, 0xc00053a420, 0x206cd00, 0xc000f8ea80, 0x0, 0x1d40b11)\n\t/home/circleci/project/pkg/store/storepb/rpc.pb.go:856 +0x113\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors.StreamServerInterceptor.func1(0x1c41d20, 0xc00053a420, 0x206cd00, 0xc000f8ea20, 0xc0005888b8, 0x1dd8468, 0x19f7a01, 0xc000f8ea20)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/server.go:35 +0x25d\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2.ChainStreamServer.func1.1.1(0x1c41d20, 0xc00053a420, 0x206cd00, 0xc000f8ea20, 0x0, 0x1d40b11)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/chain.go:51 +0x5f\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors.StreamServerInterceptor.func1(0x1c41d20, 0xc00053a420, 0x206cc70, 0xc00072c2e0, 0xc0005888b8, 0xc00072c240, 0x205d470, 0xc000f8e9c0)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/server.go:35 +0x25d\ngithub.com/thanos-io/thanos/pkg/tracing.StreamServerInterceptor.func1(0x1c41d20, 0xc00053a420, 0x206cd00, 0xc000f8e990, 0xc0005888b8, 0xc00072c240, 0x205d401, 0xc000f8e990)\n\t/home/circleci/project/pkg/tracing/grpc.go:42 +0x139\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2.ChainStreamServer.func1.1.1(0x1c41d20, 0xc00053a420, 0x206cd00, 0xc000f8e990, 0x0, 0x1d40b11)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/chain.go:51 +0x5f\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors.StreamServerInterceptor.func1(0x1c41d20, 0xc00053a420, 0x206cd90, 0xc0005888d0, 0xc0005888b8, 0xc00072c260, 0xc00022ae01, 0xc0005888d0)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/server.go:35 +0x25d\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2.ChainStreamServer.func1.1.1(0x1c41d20, 0xc00053a420, 0x206cd90, 0xc0005888d0, 0x14, 0xc000c2e3c0)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/chain.go:51 +0x5f\ngithub.com/grpc-ecosystem/go-grpc-prometheus.(*ServerMetrics).StreamServerInterceptor.func1(0x1c41d20, 0xc00053a420, 0x206d138, 0xc0006100c0, 0xc0005888b8, 0xc00072c280, 0xc00072c2c0, 0x20)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-prometheus@v1.2.0/server_metrics.go:121 +0xef\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2.ChainStreamServer.func1.1.1(0x1c41d20, 0xc00053a420, 0x206d138, 0xc0006100c0, 0x20, 0x7f0eaa5ce108)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/chain.go:51 +0x5f\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery.StreamServerInterceptor.func1(0x1c41d20, 0xc00053a420, 0x206d138, 0xc0006100c0, 0xc0005888b8, 0xc00072c2a0, 0x0, 0x0)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/interceptors/recovery/interceptors.go:45 +0xae\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2.ChainStreamServer.func1.1.1(0x1c41d20, 0xc00053a420, 0x206d138, 0xc0006100c0, 0xc000716c70, 0x40e2f8)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/chain.go:51 +0x5f\ngithub.com/grpc-ecosystem/go-grpc-middleware/v2.ChainStreamServer.func1(0x1c41d20, 0xc00053a420, 0x206d138, 0xc0006100c0, 0xc0005888b8, 0x1dd8468, 0x205d470, 0xc000f8e8d0)\n\t/home/circleci/go/pkg/mod/github.com/grpc-ecosystem/go-grpc-middleware/v2@v2.0.0-rc.2.0.20201207153454-9f6bf00c00a7/chain.go:60 +0xd1\ngoogle.golang.org/grpc.(*Server).processStreamingRPC(0xc0001ffa00, 0x2077578, 0xc000aaa300, 0xc000de4300, 0xc000472720, 0x2df5e40, 0x0, 0x0, 0x0)\n\t/home/circleci/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1336 +0x535\ngoogle.golang.org/grpc.(*Server).handleStream(0xc0001ffa00, 0x2077578, 0xc000aaa300, 0xc000de4300, 0x0)\n\t/home/circleci/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:1409 +0xc68\ngoogle.golang.org/grpc.(*Server).serveStreams.func1.1(0xc000aa8180, 0xc0001ffa00, 0x2077578, 0xc000aaa300, 0xc000de4300)\n\t/home/circleci/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:746 +0xab\ncreated by google.golang.org/grpc.(*Server).serveStreams.func1\n\t/home/circleci/go/pkg/mod/google.golang.org/grpc@v1.29.1/server.go:744 +0xa5\n
```

did I miss some config ? or there is a bug? 


Discussion:
This seems like a bug, thanks for testing the newest version! Let me look into it. Not sure how this could happen though if we are always allocating just enough memory :thinking: 
I think I found it, give me a moment
Found another problem https://github.com/thanos-io/thanos/issues/4595 wonder if it's related. cc @yeya24 



Patch:
diff --git a/pkg/store/storepb/custom.go b/pkg/store/storepb/custom.go
index 6c806cb2b47..efe711d593f 100644
--- a/pkg/store/storepb/custom.go
+++ b/pkg/store/storepb/custom.go
@@ -497,32 +497,45 @@ func (m *SeriesResponse) Marshal() (dAtA []byte, err error) {
 
 	var respBuf []byte
 
-	// No pool defined, allocate directly.
+	// Slow path with no sync.Pool.
 	if m.respPool == nil {
 		respBuf = make([]byte, size)
-	} else {
-		if m.respBuf == nil {
-			poolBuf := m.respPool.Get()
-			if poolBuf == nil {
-				respBuf = make([]byte, size)
-				m.respBuf = &respBuf
-			} else {
-				m.respBuf = poolBuf.(*[]byte)
-				respBuf = *m.respBuf
-			}
+
+		n, err := m.MarshalToSizedBuffer(respBuf)
+		if err != nil {
+			return nil, err
+		}
+		return respBuf[len(respBuf)-n:], nil
+	}
+
+	// Fast path with sync.Pool.
+	// m.respBuf must not be nil so that it would be returned to the pool.
+
+	// If no pre-allocated buffer has been passed then try to get a new one.
+	if m.respBuf == nil {
+		poolBuf := m.respPool.Get()
+		// No previous buffer found in the pool, try to allocate.
+		if poolBuf == nil {
+			respBuf = make([]byte, size)
 		} else {
-			if cap(*m.respBuf) < size {
-				if m.respPool != nil {
-					m.respPool.Put(m.respBuf)
-				}
-				respBuf = make([]byte, size)
-				m.respBuf = &respBuf
-			} else {
-				respBuf = *m.respBuf
-			}
+			// Found something, let's see if it is big enough.
+			respBuf = *(poolBuf.(*[]byte))
 		}
+	} else {
+		respBuf = *m.respBuf
+	}
+
+	// Last sanity check of the size before the marshaling.
+	if cap(respBuf) < size {
+		if m.respPool != nil {
+			m.respPool.Put(&respBuf)
+		}
+		respBuf = make([]byte, size)
 	}
+	m.respBuf = &respBuf
 
+	// Possibly trim it so that there wouldn't be left-over "garbage" in the slice.
+	// TODO: check if it is needed to always trim this.
 	marshalBuf := respBuf[:size]
 	n, err := m.MarshalToSizedBuffer(marshalBuf)
 	if err != nil {


Test patch:
diff --git a/pkg/store/storepb/custom_test.go b/pkg/store/storepb/custom_test.go
index 35901d52bce..db4d34f2d0b 100644
--- a/pkg/store/storepb/custom_test.go
+++ b/pkg/store/storepb/custom_test.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"path/filepath"
 	"sort"
+	"sync"
 	"testing"
 
 	"github.com/pkg/errors"
@@ -525,3 +526,47 @@ func TestMatchersToString_Translate(t *testing.T) {
 
 	}
 }
+
+// Tests whether the Marshal() function properly checks the size
+// of a slice returned from a sync.Pool.
+// Regression test against https://github.com/thanos-io/thanos/issues/4591.
+func TestMarshalChecksSize(t *testing.T) {
+	var s Series
+
+	rawData := []rawSeries{
+		{
+			lset: labels.Labels{labels.Label{Name: "a", Value: "c"}},
+			chunks: [][]sample{
+				{{t: 11, v: 11}, {t: 12, v: 12}, {t: 13, v: 13}, {t: 14, v: 14}},
+				{{t: 1, v: 1}, {t: 2, v: 2}, {t: 3, v: 3}, {t: 4, v: 4}},
+				{{t: 20, v: 20}, {t: 21, v: 21}, {t: 22, v: 22}, {t: 24, v: 24}},
+				{{t: 11, v: 11}, {t: 12, v: 12}, {t: 13, v: 13}, {t: 14, v: 14}},
+				{{t: 15, v: 15}, {t: 16, v: 16}, {t: 17, v: 17}, {t: 18, v: 18}},
+				{{t: 20, v: 20}, {t: 21, v: 21}, {t: 22, v: 23}, {t: 24, v: 24}},
+				{{t: 11, v: 11}, {t: 12, v: 12}, {t: 13, v: 13}, {t: 14, v: 14}},
+			},
+		},
+	}
+
+	listSS := newListSeriesSet(t, rawData)
+	lset, chks := listSS.At()
+
+	s.Chunks = chks
+	s.Labels = labelpb.ZLabelsFromPromLabels(lset)
+
+	resp := NewSeriesResponse(&s)
+
+	smallPool := sync.Pool{
+		New: func() interface{} {
+			b := make([]byte, 1)
+			return &b
+		},
+	}
+	resp.respPool = &smallPool
+
+	d, err := resp.Marshal()
+	testutil.Ok(t, err)
+	testutil.Assert(t, d != nil)
+
+	resp.Close()
+}