Repository: ChainSafe/lodestar (2025-08-14T06:47:52Z)

Issue:
[fulu] head sync failed due to retrying with an more up-to-date peer 
### Describe the bug

this could be the cause of #8183 

```
grep -e "Error: Unmatched dataColumnSidecars," -rn beacon-2025-08-12.log  | wc -l
1380
```

I investigated one example


```
613490:Aug-12 02:57:44.895[sync]          verbose: Batch download error id=Head-8, startEpoch=852, status=Downloading, peer=16...gucAHg - Unmatched dataColumnSidecars, blocks=19, blobs=108 lastMatchedSlot=27286, pending dataColumnSidecars slots=27287 27287 27287 27287 27287 27287
```

which lead to

```
 613449 Aug-12 02:57:44.892[sync]            ^[[34mdebug^[[39m: ByRange requests beaconBlocksRequest={"start_slot":"27264","count":"32","step":"1"}, dataColumnRequest={"s        tart_slot":"27264","count":"32","columns":["50","69","81","85","94","124"]}, allBlocks(19)=27264 27265 27266 27267 27269 27270 27271 27272 27273 27274 27275 27277         27278 27279 27280 27282 27283 27285 27286, allDataColumnSidecars(108)=27264:50 27264:69 27264:81 27264:85 27264:94 27264:124 27265:50 27265:69 27265:81 27265:85         27265:94 27265:124 27266:50 27266:69 27266:81 27266:85 27266:94 27266:124 27267:50 27267:69 27267:81 27267:85 27267:94 27267:124 27269:50 27269:69 27269:81 27269:        85 27269:94 27269:124 27271:50 27271:69 27271:81 27271:85 27271:94 27271:124 27272:50 27272:69 27272:81 27272:85 27272:94 27272:124 27273:50 27273:69 27273:81 272        73:85 27273:94 27273:124 27274:50 27274:69 27274:81 27274:85 27274:94 27274:124 27275:50 27275:69 27275:81 27275:85 27275:94 27275:124 27277:50 27277:69 27277:81         27277:85 27277:94 27277:124 27279:50 27279:69 27279:81 27279:85 27279:94 27279:124 27280:50 27280:69 27280:81 27280:85 27280:94 27280:124 27282:50 27282:69 27282:        81 27282:85 27282:94 27282:124 27283:50 27283:69 27283:81 27283:85 27283:94 27283:124 27285:50 27285:69 27285:81 27285:85 27285:94 27285:124 27286:50 27286:69 272        86:81 27286:85 27286:94 27286:124 27287:50 27287:69 27287:81 27287:85 27287:94 27287:124, peerColumns=23 37 50 69 81 85 94 124, peerId=16Uiu2HAmJAC6f8AC8i9yj1ncPE        z7XBBMT67ZGw7b2gkRxxgucAHg, peerClient=Prysm, prevPartialDownload=true
```

the issue is there was a previous partial download and blocks were fetched from another peer. At that time clock slot is 27286
then at this retry, it does not download block again, and peer responded blobs/data columns for block slot 27287 (which could be the head of that peer, clock slot is ticked to 27287)



### Expected behavior

for unconsumed data columns, only throw error if it's finalized range sync
do not throw error for head range sync

### Steps to reproduce

_No response_

### Additional context

_No response_

### Operating system

Linux

### Lodestar version or commit hash

unstable, fusaka-devnet-4
[fulu] head sync to match DataColumnSidecars using block header root
### Problem description

right now when retrying a batch we only compare slot which works fine for finality chain but not for head chain

### Solution description

during unfinality time there could be different head chains so we need to compare by block header instead

### Additional context

_No response_


Discussion:

Patch:
diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts
index 27dc5e1d15ff..5497f4500541 100644
--- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts
+++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts
@@ -28,12 +28,15 @@ import {
 } from "../../chain/blocks/types.js";
 import {getEmptyBlockInputCacheEntry} from "../../chain/seenCache/seenGossipBlockInput.js";
 import {Metrics} from "../../metrics/index.js";
+import {RangeSyncType} from "../../sync/utils/remoteSyncType.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {INetwork} from "../interface.js";
 import {PeerSyncMeta} from "../peers/peersData.js";
 import {PeerAction} from "../peers/score/interface.js";
 
 export type PartialDownload = null | {blocks: BlockInput[]; pendingDataColumns: number[]};
+export const SyncSourceByRoot = "ByRoot" as const;
+export type SyncSource = RangeSyncType | typeof SyncSourceByRoot;
 
 /**
  * Download blocks and blobs (prefulu) or data columns (fulu) by range.
@@ -48,6 +51,7 @@ export async function beaconBlocksMaybeBlobsByRange(
   request: phase0.BeaconBlocksByRangeRequest,
   currentEpoch: Epoch,
   partialDownload: PartialDownload,
+  syncSource: SyncSource,
   metrics: Metrics | null,
   logger?: Logger
 ): Promise<{blocks: BlockInput[]; pendingDataColumns: null | number[]}> {
@@ -104,7 +108,8 @@ export async function beaconBlocksMaybeBlobsByRange(
         allBlobSidecars,
         endSlot,
         BlockSource.byRange,
-        BlobsSource.byRange
+        BlobsSource.byRange,
+        syncSource
       );
       return {blocks, pendingDataColumns: null};
     }
@@ -181,6 +186,7 @@ export async function beaconBlocksMaybeBlobsByRange(
       DataColumnsSource.byRange,
       partialDownload,
       peerClient,
+      syncSource,
       metrics,
       logger
     );
@@ -219,7 +225,8 @@ export function matchBlockWithBlobs(
   allBlobSidecars: deneb.BlobSidecar[],
   endSlot: Slot,
   blockSource: BlockSource,
-  blobsSource: BlobsSource
+  blobsSource: BlobsSource,
+  syncSource: SyncSource
 ): BlockInput[] {
   const blockInputs: BlockInput[] = [];
   let blobSideCarIndex = 0;
@@ -238,12 +245,27 @@ export function matchBlockWithBlobs(
     } else {
       const blobSidecars: deneb.BlobSidecar[] = [];
 
-      let blobSidecar: deneb.BlobSidecar;
-      while (
-        // biome-ignore lint/suspicious/noAssignInExpressions: <explanation>
-        (blobSidecar = allBlobSidecars[blobSideCarIndex])?.signedBlockHeader.message.slot === block.data.message.slot
-      ) {
-        blobSidecars.push(blobSidecar);
+      const blockRoot = config.getForkTypes(block.data.message.slot).BeaconBlock.hashTreeRoot(block.data.message);
+      const matchBlob = (blobSidecar?: deneb.BlobSidecar): boolean => {
+        if (blobSidecar === undefined) {
+          return false;
+        }
+
+        if (syncSource === RangeSyncType.Head || syncSource === SyncSourceByRoot) {
+          return (
+            Buffer.compare(
+              ssz.phase0.BeaconBlockHeader.hashTreeRoot(blobSidecar.signedBlockHeader.message),
+              blockRoot
+            ) === 0
+          );
+        }
+
+        // For finalized range sync, we can just match by slot
+        return blobSidecar.signedBlockHeader.message.slot === block.data.message.slot;
+      };
+
+      while (matchBlob(allBlobSidecars[blobSideCarIndex])) {
+        blobSidecars.push(allBlobSidecars[blobSideCarIndex]);
         lastMatchedSlot = block.data.message.slot;
         blobSideCarIndex++;
       }
@@ -298,6 +320,7 @@ export function matchBlockWithDataColumns(
   dataColumnsSource: DataColumnsSource,
   prevPartialDownload: null | PartialDownload,
   peerClient: string,
+  syncSource: SyncSource,
   metrics: Metrics | null,
   logger?: Logger
 ): BlockInput[] {
@@ -321,7 +344,25 @@ export function matchBlockWithDataColumns(
       throw Error(`Invalid block forkSeq=${forkSeq} < ForSeq.fulu for matchBlockWithDataColumns`);
     }
     const dataColumnSidecars: fulu.DataColumnSidecar[] = [];
-    while (allDataColumnSidecars[dataColumnSideCarIndex]?.signedBlockHeader.message.slot === block.data.message.slot) {
+    const blockRoot = config.getForkTypes(block.data.message.slot).BeaconBlock.hashTreeRoot(block.data.message);
+    const matchDataColumnSidecar = (dataColumnSidecar?: fulu.DataColumnSidecar): boolean => {
+      if (dataColumnSidecar === undefined) {
+        return false;
+      }
+
+      if (syncSource === RangeSyncType.Head || syncSource === SyncSourceByRoot) {
+        return (
+          Buffer.compare(
+            ssz.phase0.BeaconBlockHeader.hashTreeRoot(dataColumnSidecar.signedBlockHeader.message),
+            blockRoot
+          ) === 0
+        );
+      }
+
+      // For finalized range sync, we can just match by slot
+      return dataColumnSidecar.signedBlockHeader.message.slot === block.data.message.slot;
+    };
+    while (matchDataColumnSidecar(allDataColumnSidecars[dataColumnSideCarIndex])) {
       dataColumnSidecars.push(allDataColumnSidecars[dataColumnSideCarIndex]);
       lastMatchedSlot = block.data.message.slot;
       dataColumnSideCarIndex++;
@@ -341,7 +382,10 @@ export function matchBlockWithDataColumns(
     });
     if (blobKzgCommitmentsLen === 0) {
       if (dataColumnSidecars.length > 0) {
-        network.reportPeer(peerId, PeerAction.LowToleranceError, "Missing or mismatching dataColumnSidecars");
+        // only penalize peer with Finalized range sync or "ByRoot" sync source
+        if (syncSource !== RangeSyncType.Head) {
+          network.reportPeer(peerId, PeerAction.LowToleranceError, "Missing or mismatching dataColumnSidecars");
+        }
         throw Error(
           `Missing or mismatching dataColumnSidecars from peerId=${peerId} for blockSlot=${block.data.message.slot} with blobKzgCommitmentsLen=0 dataColumnSidecars=${dataColumnSidecars.length}>0`
         );
@@ -380,7 +424,10 @@ export function matchBlockWithDataColumns(
             peerClient,
           }
         );
-        network.reportPeer(peerId, PeerAction.LowToleranceError, "Missing or mismatching dataColumnSidecars");
+        // only penalize peer with Finalized range sync or "ByRoot" sync source
+        if (syncSource !== RangeSyncType.Head) {
+          network.reportPeer(peerId, PeerAction.LowToleranceError, "Missing or mismatching dataColumnSidecars");
+        }
         throw Error(
           `Missing or mismatching dataColumnSidecars from peerId=${peerId} for blockSlot=${block.data.message.slot} blobKzgCommitmentsLen=${blobKzgCommitmentsLen} with numColumns=${sampledColumns.length} dataColumnSidecars=${dataColumnSidecars.length} requestedColumnsPresent=${requestedColumnsPresent} received dataColumnIndexes=${dataColumnIndexes.join(" ")} requested=${requestedColumns.join(" ")}`
         );
@@ -434,12 +481,13 @@ export function matchBlockWithDataColumns(
     }
   }
 
-  // If there are still unconsumed blobs this means that the response was inconsistent
-  // and matching was wrong and hence we should throw error
+  // for head sync, there could be unconsumed data column sidecars because the retried peers may have higher head
   if (
     allDataColumnSidecars[dataColumnSideCarIndex] !== undefined &&
     // If there are no data columns, the data columns request can give 1 block outside the requested range
-    allDataColumnSidecars[dataColumnSideCarIndex].signedBlockHeader.message.slot <= endSlot
+    allDataColumnSidecars[dataColumnSideCarIndex].signedBlockHeader.message.slot <= endSlot &&
+    // only penalize peer with Finalized range sync or "ByRoot" sync source
+    syncSource !== RangeSyncType.Head
   ) {
     network.reportPeer(peerId, PeerAction.LowToleranceError, "Unmatched dataColumnSidecars");
     throw Error(
@@ -448,7 +496,7 @@ export function matchBlockWithDataColumns(
       } lastMatchedSlot=${lastMatchedSlot}, pending dataColumnSidecars slots=${allDataColumnSidecars
         .slice(dataColumnSideCarIndex)
         .map((blb) => blb.signedBlockHeader.message.slot)
-        .join(" ")}`
+        .join(" ")} endSlot=${endSlot}, peerId=${peerId}, peerClient=${peerClient}`
     );
   }
   logger?.debug("matched BlockWithDataColumns", {
diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts
index 7bed3303fb71..f24febac8c7e 100644
--- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts
+++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts
@@ -29,7 +29,12 @@ import {computeInclusionProof, kzgCommitmentToVersionedHash} from "../../util/bl
 import {getDataColumnsFromExecution} from "../../util/dataColumns.js";
 import {PeerIdStr} from "../../util/peerId.js";
 import {INetwork} from "../interface.js";
-import {PartialDownload, matchBlockWithBlobs, matchBlockWithDataColumns} from "./beaconBlocksMaybeBlobsByRange.js";
+import {
+  PartialDownload,
+  SyncSourceByRoot,
+  matchBlockWithBlobs,
+  matchBlockWithDataColumns,
+} from "./beaconBlocksMaybeBlobsByRange.js";
 
 // keep 1 epoch of stuff, assmume 16 blobs
 const MAX_ENGINE_GETBLOBS_CACHE = 32 * 16;
@@ -125,7 +130,8 @@ export async function beaconBlocksMaybeBlobsByRoot(
       allBlobSidecars,
       Infinity,
       BlockSource.byRoot,
-      BlobsSource.byRoot
+      BlobsSource.byRoot,
+      SyncSourceByRoot
     );
     blockInputs = [...blockInputs, ...blockInputWithBlobs];
   }
@@ -175,6 +181,7 @@ export async function beaconBlocksMaybeBlobsByRoot(
       DataColumnsSource.byRoot,
       partialDownload,
       peerClient,
+      SyncSourceByRoot,
       metrics,
       logger
     );
diff --git a/packages/beacon-node/src/sync/range/chain.ts b/packages/beacon-node/src/sync/range/chain.ts
index 674e201b88c5..1f9aea719562 100644
--- a/packages/beacon-node/src/sync/range/chain.ts
+++ b/packages/beacon-node/src/sync/range/chain.ts
@@ -44,7 +44,8 @@ export type SyncChainFns = {
   downloadBeaconBlocksByRange: (
     peer: PeerSyncMeta,
     request: phase0.BeaconBlocksByRangeRequest,
-    partialDownload: PartialDownload
+    partialDownload: PartialDownload,
+    syncType: RangeSyncType
   ) => Promise<{blocks: BlockInput[]; pendingDataColumns: null | number[]}>;
   /** Report peer for negative actions. Decouples from the full network instance */
   reportPeer: (peer: PeerIdStr, action: PeerAction, actionName: string) => void;
@@ -449,7 +450,9 @@ export class SyncChain {
       const partialDownload = batch.startDownloading(peer.peerId);
 
       // wrapError ensures to never call both batch success() and batch error()
-      const res = await wrapError(this.downloadBeaconBlocksByRange(peer, batch.request, partialDownload));
+      const res = await wrapError(
+        this.downloadBeaconBlocksByRange(peer, batch.request, partialDownload, this.syncType)
+      );
 
       if (!res.err) {
         const downloadSuccessOutput = batch.downloadingSuccess(res.result);
diff --git a/packages/beacon-node/src/sync/range/range.ts b/packages/beacon-node/src/sync/range/range.ts
index c58dcb1d335d..45e90f68c5c1 100644
--- a/packages/beacon-node/src/sync/range/range.ts
+++ b/packages/beacon-node/src/sync/range/range.ts
@@ -203,7 +203,8 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
   private downloadBeaconBlocksByRange: SyncChainFns["downloadBeaconBlocksByRange"] = async (
     peer,
     request,
-    partialDownload
+    partialDownload,
+    syncType: RangeSyncType
   ) => {
     return beaconBlocksMaybeBlobsByRange(
       this.config,
@@ -212,6 +213,7 @@ export class RangeSync extends (EventEmitter as {new (): RangeSyncEmitter}) {
       request,
       this.chain.clock.currentEpoch,
       partialDownload,
+      syncType,
       this.metrics,
       this.logger
     );


Test patch:
diff --git a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts
index 82361d125eb6..ba3993b7b23c 100644
--- a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts
+++ b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts
@@ -7,6 +7,7 @@ import {BlobsSource, BlockSource, getBlockInput} from "../../../src/chain/blocks
 import {ZERO_HASH} from "../../../src/constants/constants.js";
 import {INetwork} from "../../../src/network/interface.js";
 import {beaconBlocksMaybeBlobsByRange} from "../../../src/network/reqresp/index.js";
+import {RangeSyncType} from "../../../src/sync/utils/remoteSyncType.js";
 import {CustodyConfig} from "../../../src/util/dataColumns.js";
 
 describe.skip("beaconBlocksMaybeBlobsByRange", () => {
@@ -126,6 +127,7 @@ describe.skip("beaconBlocksMaybeBlobsByRange", () => {
         rangeRequest,
         0,
         null,
+        RangeSyncType.Finalized,
         null
       );
       expect(response).toEqual(expectedResponse);
diff --git a/packages/beacon-node/test/unit/sync/range/chain.test.ts b/packages/beacon-node/test/unit/sync/range/chain.test.ts
index e44519ad1640..7e4b03b11acd 100644
--- a/packages/beacon-node/test/unit/sync/range/chain.test.ts
+++ b/packages/beacon-node/test/unit/sync/range/chain.test.ts
@@ -216,9 +216,9 @@ function logSyncChainFns(logger: Logger, fns: SyncChainFns): SyncChainFns {
       logger.debug("mock processChainSegment", {blocks: blocks.map((b) => b.block.message.slot).join(",")});
       return fns.processChainSegment(blocks, syncType);
     },
-    downloadBeaconBlocksByRange(peer, request, _partialDownload) {
+    downloadBeaconBlocksByRange(peer, request, _partialDownload, syncType) {
       logger.debug("mock downloadBeaconBlocksByRange", request);
-      return fns.downloadBeaconBlocksByRange(peer, request, _partialDownload);
+      return fns.downloadBeaconBlocksByRange(peer, request, _partialDownload, syncType);
     },
     getConnectedPeerSyncMeta(peerId) {
       logger.debug("mock getConnectedPeerSyncMeta", peerId);