Repository: celestiaorg/celestia-app

Issue:
Error: writing square: square size 16 is too small to fit all blobs
validator's have observed a bug in mocha that caused a few nodes to panic

> writing square: square size 16 is too small to fit all blobs

This occured during `PrepareProposal`, therefore its possible that this isn't a non-determinism issue.

from a very helpful discord validator:
```
May 08 11:39:28 celestia-appd[142019]: 11:39AM ERR CONSENSUS FAILURE!!! err="writing square: square size 16 is too small to fit all blobs" module=consensus stack="goroutine 3792 [running]:\nruntime/debug.Stack()\n\t/home/space/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.23.6.linux-amd64/src/runtime/debug/stack.go:26 +0x5e\ngithub.com/tendermint/tendermint/consensus.(*State).receiveRoutine.func2()\n\t/home/space/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:758 +0x46\npanic({0x254ae80?, 0xc4b4adb800?})\n\t/home/space/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.23.6.linux-amd64/src/runtime/panic.go:785 +0x132\ngithub.com/celestiaorg/celestia-app/v3/app.(*App).PrepareProposal(0xc023910a88, {0xc4b4ac4000, 0x7fd1c9, {0xc255fe4b57, 0x7}, 0x5cb2c6, {0x31c74935, 0xedfae8eeb, 0x0}})\n\t/home/space/celestia-app/app/prepare_proposal.go:79 +0xa71\ngithub.com/tendermint/tendermint/abci/client.(*localClient).PrepareProposalSync(0xc00179b200, {0xc4b4ac4000, 0x7fd1c9, {0xc255fe4b57, 0x7}, 0x5cb2c6, {0x31c74935, 0xedfae8eeb, 0x0}})\n\t/home/space/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/abci/client/local_client.go:359 +0x155\ngithub.com/tendermint/tendermint/proxy.(*appConnConsensus).PrepareProposalSync(0xc49cd90580?, {0xc4b4ac4000, 0x7fd1c9, {0xc255fe4b57, 0x7}, 0x5cb2c6, {0x31c74935, 0xedfae8eeb, 0x0}})\n\t/home/space/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/proxy/app_conn.go:102 +0x66\ngithub.com/tendermint/tendermint/state.(*BlockExecutor).CreateProposalBlock(_, _, {{{0xb, 0x3}, {0xc255fe4b50, 0x7}}, {0xc255fe4b57, 0x7}, 0x1, 0x5cb2c5, ...}, ...)\n\t/home/space/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/state/execution.go:128 +0x2f3\ngithub.com/tendermint/tendermint/consensus.(*State).createProposalBlock(0xc0007ddc08)\n\t/home/space/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:1262 +0x27b\ngithub.com/tendermint/tend
```

Discussion:
@JB273: P-OPS Team validator experienced this ~1h 20m ago:
```
4:34PM INF minted coins from module account amount=13343304utia from=mint module=x/bank
4:34PM INF executed block height=6190308 module=state num_invalid_txs=0 num_valid_txs=6
4:34PM INF commit synced commit=436F6D6D697449447B5B3130322031323520333920332032323420323433203137392031323220313330203230382031353220313020363220333420313637203130203232342031383320313239203131203131>4:34PM INF committed state app_hash=667D2703E0F3B37A82D0980A3E22A70AE0B7810B6F41529B3AA1D319CC545B4B height=6190308 module=state num_txs=6
4:34PM INF indexed block events height=6190308 module=txindex
4:34PM INF Timed out dur=4038.741794 height=6190309 module=consensus round=0 step=1
4:34PM ERR CONSENSUS FAILURE!!! err="writing square: square size 16 is too small to fit all blobs" module=consensus stack="goroutine 96040 [running]:\nruntime/debug.Stack()\n\t/home/pops/go/pkg/mod/go>4:34PM INF service stop impl={"Logger":{}} module=consensus msg={} wal=/home/pops/.celestia-app/data/cs.wal/wal
4:34PM INF service stop impl={"Dir":"/home/pops/.celestia-app/data/cs.wal","Head":{"ID":"B2O5iBNtLWBN:/home/pops/.celestia-app/data/cs.wal/wal","Path":"/home/pops/.celestia-app/data/cs.wal/wal"},"ID":>4:34PM INF Timed out dur=3500 height=6190309 module=consensus round=0 step=3
4:34PM INF Ensure peers module=pex numDialing=0 numInPeers=40 numOutPeers=9 numToDial=1
4:35PM ERR Connection failed @ sendRoutine conn={"Logger":{}} err="pong timeout" module=p2p peer={"id":"45504fb31eb97ea8778c920701fc8076e568a9cd","ip":"188.214.133.100","port":60078}
4:35PM INF service stop impl={"Logger":{}} module=p2p msg={} peer={"id":"45504fb31eb97ea8778c920701fc8076e568a9cd","ip":"188.214.133.100","port":60078}
4:35PM ERR Stopping peer for error err="pong timeout" module=p2p peer={"Data":{},"Logger":{}}
4:35PM INF service stop impl={"Data":{},"Logger":{}} module=p2p msg={} peer={"id":"45504fb31eb97ea8778c920701fc8076e568a9cd","ip":"188.214.133.100","port":60078}
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM INF Inbound Peer rejected err="auth failure: secret conn failed: EOF" module=p2p numPeers=48
4:35PM ERR Connection failed @ sendRoutine conn={"Logger":{}} err="pong timeout" module=p2p peer={"id":"d7fc95caa0450605b6cbc0a8ab47e3f20992cfd1","ip":"46.4.115.61","port":36754}
4:35PM INF service stop impl={"Logger":{}} module=p2p msg={} peer={"id":"d7fc95caa0450605b6cbc0a8ab47e3f20992cfd1","ip":"46.4.115.61","port":36754}
4:35PM ERR Stopping peer for error err="pong timeout" module=p2p peer={"Data":{},"Logger":{}}
```@easy2stake: This is happening apparently random, to various validators.
Not the same block.

Sharing our logs:
```
8:11PM INF executed block height=6176627 module=state num_invalid_txs=0 num_valid_txs=6
8:11PM INF commit synced commit=436F6D6D697449447B5B3235312031343220352031353120313734203231322031373220323334203235322031303620323033203620323237203233382031353520353920313930203238203135372032342034332032333720313738203133362033362031322031303020353620383620353720313332203132315D3A3545334637337D
8:11PM INF committed state app_hash=FB8E0597AED4ACEAFC6ACB06E3EE9B3BBE1C9D182BEDB288240C643856398479 height=6176627 module=state num_txs=6
8:11PM INF indexed block events height=6176627 module=txindex
8:11PM INF Timed out dur=4130.760564 height=6176628 module=consensus round=0 step=1
8:11PM ERR CONSENSUS FAILURE!!! err="writing square: square size 16 is too small to fit all blobs" module=consensus stack="goroutine 354 [running]:\nruntime/debug.Stack()\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.23.6.linux-amd64/src/runtime/debug/stack.go:26 +0x5e\ngithub.com/tendermint/tendermint/consensus.(*State).receiveRoutine.func2()\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:758 +0x46\npanic({0x254ae80?, 0xc01af53440?})\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.23.6.linux-amd64/src/runtime/panic.go:785 +0x132\ngithub.com/celestiaorg/celestia-app/v3/app.(*App).PrepareProposal(0xc00129c008, {0xc05a815280, 0x7fd1c9, {0xc005846fe7, 0x7}, 0x5e3f74, {0x315eeee8, 0xedfb6c5c5, 0x0}})\n\t/home/celestia/celestia-app/app/prepare_proposal.go:79 +0xa71\ngithub.com/tendermint/tendermint/abci/client.(*localClient).PrepareProposalSync(0xc0016bea80, {0xc05a815280, 0x7fd1c9, {0xc005846fe7, 0x7}, 0x5e3f74, {0x315eeee8, 0xedfb6c5c5, 0x0}})\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/abci/client/local_client.go:359 +0x155\ngithub.com/tendermint/tendermint/proxy.(*appConnConsensus).PrepareProposalSync(0xc03f178d00?, {0xc05a815280, 0x7fd1c9, {0xc005846fe7, 0x7}, 0x5e3f74, {0x315eeee8, 0xedfb6c5c5, 0x0}})\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/proxy/app_conn.go:102 +0x66\ngithub.com/tendermint/tendermint/state.(*BlockExecutor).CreateProposalBlock(_, _, {{{0xb, 0x3}, {0xc005846fe0, 0x7}}, {0xc005846fe7, 0x7}, 0x1, 0x5e3f73, ...}, ...)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/state/execution.go:128 +0x2f3\ngithub.com/tendermint/tendermint/consensus.(*State).createProposalBlock(0xc0012aee08)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:1262 +0x27b\ngithub.com/tendermint/tendermint/consensus.(*State).defaultDecideProposal(0xc0012aee08, 0x5e3f74, 0x0)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:1176 +0xda\ngithub.com/tendermint/tendermint/consensus.(*State).enterPropose(0xc0012aee08, 0x5e3f74, 0x0)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:1155 +0x83e\ngithub.com/tendermint/tendermint/consensus.(*State).enterNewRound(0xc0012aee08, 0x5e3f74, 0x0)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:1074 +0xb58\ngithub.com/tendermint/tendermint/consensus.(*State).handleTimeout(0xc0012aee08, {0x5ea145?, 0xc002674300?, 0x2674320?, 0xc0?}, {0x5e3f74, 0x0, 0x1, {0x13808254, 0xedfb6c5ca, ...}, ...})\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:943 +0x8c7\ngithub.com/tendermint/tendermint/consensus.(*State).receiveRoutine(0xc0012aee08, 0x0)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:825 +0x6bf\ncreated by github.com/tendermint/tendermint/consensus.(*State).OnStart in goroutine 164\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.51.0-tm-v0.34.35/consensus/state.go:392 +0x13b\n"
8:11PM INF service stop impl={"Logger":{}} module=consensus msg={} wal=/home/celestia/.celestia-app/data/cs.wal/wal
8:11PM INF service stop impl={"Dir":"/home/celestia/.celestia-app/data/cs.wal","Head":{"ID":"ZlUQypPyovmR:/home/celestia/.celestia-app/data/cs.wal/wal","Path":"/home/celestia/.celestia-app/data/cs.wal/wal"},"ID":"group:ZlUQypPyovmR:/home/celestia/.celestia-app/data/cs.wal/wal","Logger":{}} module=consensus msg={} wal=/home/celestia/.celestia-app/data/cs.wal/wal
8:11PM INF Timed out dur=3500 height=6176628 module=consensus round=0 step=3```
@rootulp: I agree this is a problem and we should investigate it but I'm working on v4 bugs and haven't looked into this yet.@ninabarbakadze: I queried all transactions from blocks that experienced the panic and tried to pass them into the square builder. Had no issues there. Then I tried to pass them into `PrepareProposal` but I'm having issues with signers/fee payers not being part of the keyring. If I pass the addresses into the keyring will that work? @evan-forbes I'm a bit rusty on that. Will continue on this tomorrow. @evan-forbes: We need a snapshot of the state in order to check stateful things in the antehandlers unfortunately 

There might be some issue with non-determinism, so running the test 1000 time might help. but iirc when I ran this i couldn't recreate the issue with fuzzing either.

Next steps would be try and via the multiplexer and embedded binary given the number or weird issues there@rootulp: This bug was reported before we asked Mocha nodes to update to v4.0.2-mocha so I have a strong suspicion that the bug is present in v3.8.1. 

I think Nina will hit a bunch of other multiplexer bugs if she tries reproducing this with v4.0.2-mocha so would recommend trying to reproduce with v3.8.1 which was extremely reliable (with the exception of the bug in this issue).@maxzonder: validator node stalled today with `panic: writing square: square size 256 is too small to fit all blobs`, restart resolved

logs:
```
6:32AM INF received complete proposal block hash=B6FEDBD3DEBB89FD6BB49B1DA477377707BA66449FD474F7EC4DF74B46E49781 height=6850029 module=consensus
6:32AM INF finalizing commit of block hash=B6FEDBD3DEBB89FD6BB49B1DA477377707BA66449FD474F7EC4DF74B46E49781 height=6850029 module=consensus num_txs=5 root=244958E583D1CE6C650A1CF26C04657241895DC37912424BCBE5E7C73835CE86
6:32AM INF minted coins from module account amount=13171728utia from=mint module=x/bank
6:32AM INF commit synced commit=436F6D6D697449447B5B313734203732203835203231352032323320363320313632203530203635203132392031383020353720313331203137392031383620313837203136372037362037342031313820323435203130362032323020323434203534203135322038302035312034203639203839203232375D3A3638383545447D
6:32AM INF finalized block block_app_hash=AE4855D7DF3FA2324181B43983B3BABBA74C4A76F56ADCF436985033044559E3 height=6850029 module=state num_txs_res=5 num_val_updates=0
6:32AM INF executed block app_hash=AE4855D7DF3FA2324181B43983B3BABBA74C4A76F56ADCF436985033044559E3 height=6850029 module=state
6:32AM INF committed state block_app_hash=244958E583D1CE6C650A1CF26C04657241895DC37912424BCBE5E7C73835CE86 height=6850029 module=state
6:32AM INF indexed block events height=6850029 module=txindex
6:32AM INF Timed out dur=4017.772461 height=6850030 module=consensus round=0 step=RoundStepNewHeight
panic: writing square: square size 256 is too small to fit all blobs
goroutine 1417936 [running]:
github.com/celestiaorg/celestia-app/v3/app.(*App).PrepareProposal(0xc003ceea88, {0xc05cafd0c0, 0x7fffffff, {0xc063a7e220, 0x7}, 0x6885ee, {0x34a13e24, 0xedff031ec, 0x0}})
        /home/celestia/celestia-app/app/prepare_proposal.go:67 +0xbc6
github.com/tendermint/tendermint/abci/types.(*GRPCApplication).PrepareProposal(0xc004094bb0, {0xc05cafcf00?, 0xc0009a9900?}, 0xc05cafcf00)
        /home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.55.0-tm-v0.34.35/abci/types/application.go:198 +0xb2
github.com/tendermint/tendermint/abci/types._ABCIApplication_PrepareProposal_Handler({0x28531a0, 0xc004094bb0}, {0x33d9308, 0xc0105dbf50}, 0xc03fd71e80, 0x0)
        /home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.55.0-tm-v0.34.35/abci/types/types.pb.go:4379 +0x1a9
google.golang.org/grpc.(*Server).processUnaryRPC(0xc00040b200, {0x33d9308, 0xc00ad7ea50}, {0x33ec140, 0xc0040b6680}, 0xc04388bc20, 0xc0040de270, 0x49baae8, 0x0)
        /home/celestia/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1394 +0xe24
google.golang.org/grpc.(*Server).handleStream(0xc00040b200, {0x33ec140, 0xc0040b6680}, 0xc04388bc20)
        /home/celestia/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1805 +0xe8a
google.golang.org/grpc.(*Server).serveStreams.func2.1()
        /home/celestia/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1029 +0x7f
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 262
        /home/celestia/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1040 +0x11d
2025/06/27 06:32:17 Process finished with error: exit status 2
6:32AM ERR CONSENSUS FAILURE!!! err="rpc error: code = Unavailable desc = error reading from server: EOF" module=consensus stack="goroutine 3038 [running]:\nruntime/debug.Stack()\n\t/usr/local/go/src/runtime/debug/stack.go:26 +0x5e\ngithub.com/cometbft/cometbft/consensus.(*State).receiveRoutine.func2()\n\t/home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:825 +0x46\npanic({0x40300e0?, 0xc0104901b8?})\n\t/usr/local/go/src/runtime/panic.go:792 +0x132\ngithub.com/cometbft/cometbft/consensus.(*State).createProposalBlock(0xc00dbe9c08, {0x4f50160, 0x96ca860})\n\t/home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1335 +0x28f\ngithub.com/cometbft/cometbft/consensus.(*State).defaultDecideProposal(0xc00dbe9c08, 0x6885ee, 0x0)\n\t/home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1240 +0x65\ngithub.com/cometbft/cometbft/consensus.(*State).enterPropose(0xc00dbe9c08, 0x6885ee, 0x0)\n\t/home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1219 +0x82e\ngithub.com/cometbft/cometbft/consensus.(*State).enterNewRound(0xc00dbe9c08, 0x6885ee, 0x0)\n\t/home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1138 +0xb4f\ngithub.com/cometbft/cometbft/consensus.(*State).handleTimeout(0xc00dbe9c08, {0x0?, 0x0?, 0x0?, 0x0?}, {0x6885ee, 0x0, 0x1, {0x181d32e9, 0xedff031f1, ...}, ...})\n\t/home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1009 +0x8c9\ngithub.com/cometbft/cometbft/consensus.(*State).receiveRoutine(0xc00dbe9c08, 0x0)\n\t/home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:891 +0x6bf\ncreated by github.com/cometbft/cometbft/consensus.(*State).OnStart in goroutine 2838\n\t/home/celestia/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:411 +0x132\n"
6:32AM INF service stop impl=baseWAL module=consensus msg="Stopping baseWAL service" wal=/home/celestia/.celestia-app/data/cs.wal/wal
6:32AM INF service stop impl=Group module=consensus msg="Stopping Group service" wal=/home/celestia/.celestia-app/data/cs.wal/wal
6:32AM INF Timed out dur=3500 height=6850030 module=consensus round=0 step=RoundStepPropose
6:32AM INF Ensure peers module=pex numDialing=0 numInPeers=0 numOutPeers=9 numToDial=1
6:32AM INF Saving AddrBook to file book=/home/celestia/.celestia-app/config/addrbook.json module=p2p size=1100
6:33AM INF Ensure peers module=pex numDialing=0 numInPeers=0 numOutPeers=9 numToDial=1
```

`v4.0.6-mocha`@evan-forbes: thanks for the helpful info @maxzonder! we're going to add something in the next release that will capture the inputs that are causing these panics as we're having trouble replicating the issue with fuzzing. In theory, we should never be hitting this since the square builder should be filtering out blobs before it exports them

https://github.com/celestiaorg/celestia-core/pull/1992@kinrokinro: We experienced this on the latest release of the testnet (v4.0.6-mocha) on both testnet sentries. The RPC nodes seems to be fine:
```
11:59AM INF finalized block block_app_hash=45C7A3F2A52AB86D6EA8F9646C056DF0958A9E0D8EE6D49256C93C2CFDE88D88 height=6853273 module=state num_txs_res=6 num_val_updates=0
11:59AM INF executed block app_hash=45C7A3F2A52AB86D6EA8F9646C056DF0958A9E0D8EE6D49256C93C2CFDE88D88 height=6853273 module=state
11:59AM INF committed state block_app_hash=FF7FC8337351CD442F2C3DD228149AE1BBBFD9D66980584ADECA5020656D5440 height=6853273 module=state
12:00PM INF Timed out dur=3690.402894 height=6853274 module=consensus round=0 step=RoundStepNewHeight
panic: writing square: square size 256 is too small to fit all blobs
goroutine 686050 [running]:
github.com/celestiaorg/celestia-app/v3/app.(*App).PrepareProposal(0xc00122b508, {0xc02e5c3f40, 0x7fffffff, {0xc0188a3638, 0x7}, 0x68929a, {0x303d3c3c, 0xedff07ebd, 0x0}})
        /go/src/github.com/celestiaorg/celestia-app/v3/app/prepare_proposal.go:67 +0xbf6
github.com/tendermint/tendermint/abci/types.(*GRPCApplication).PrepareProposal(0xc001348980, {0x27f6500?, 0xc02d874900?}, 0xc02e5c3f00)
        /root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.55.0-tm-v0.34.35/abci/types/application.go:198 +0xc2
github.com/tendermint/tendermint/abci/types._ABCIApplication_PrepareProposal_Handler({0x27f6500, 0xc001348980}, {0x336f648, 0xc02be2b560}, 0xc022fbaf00, 0x0)
        /root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.55.0-tm-v0.34.35/abci/types/types.pb.go:4379 +0x1a9
google.golang.org/grpc.(*Server).processUnaryRPC(0xc0001c1400, {0x336f648, 0xc02be2b410}, {0x3382320, 0xc005c2e000}, 0xc025f6f560, 0xc0012a0a20, 0x49157a8, 0x0)
        /root/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1394 +0xe2b
google.golang.org/grpc.(*Server).handleStream(0xc0001c1400, {0x3382320, 0xc005c2e000}, 0xc025f6f560)
        /root/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1805 +0xe8b
google.golang.org/grpc.(*Server).serveStreams.func2.1()
        /root/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1029 +0x7f
created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 49
        /root/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1040 +0x125
2025/06/27 12:00:02 Process finished with error: exit status 2
12:00PM ERR CONSENSUS FAILURE!!! err="rpc error: code = Unavailable desc = error reading from server: EOF" module=consensus stack="goroutine 2504 [running]:\nruntime/debug.Stack()\n\t/home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.24.2.linux-amd64/src/runtime/debug/stack.go:26 +0x5e\ngithub.com/cometbft/cometbft/consensus.(*State).receiveRoutine.func2()\n\t/home/ubuntu/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:825 +0x46\npanic({0x40530e0?, 0xc013e723b8?})\n\t/home/ubuntu/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.24.2.linux-amd64/src/runtime/panic.go:792 +0x132\ngithub.com/cometbft/cometbft/consensus.(*State).createProposalBlock(0xc0024f0388, {0x4f73140, 0xabc16e0})\n\t/home/ubuntu/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1335 +0x28f\ngithub.com/cometbft/cometbft/consensus.(*State).defaultDecideProposal(0xc0024f0388, 0x68929a, 0x0)\n\t/home/ubuntu/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1240 +0x65\ngithub.com/cometbft/cometbft/consensus.(*State).enterPropose(0xc0024f0388, 0x68929a, 0x0)\n\t/home/ubuntu/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1219 +0x82e\ngithub.com/cometbft/cometbft/consensus.(*State).enterNewRound(0xc0024f0388, 0x68929a, 0x0)\n\t/home/ubuntu/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1138 +0xb4f\ngithub.com/cometbft/cometbft/consensus.(*State).handleTimeout(0xc0024f0388, {0xaea125?, 0xc002f35e28?, 0x1417ba4?, 0x0?}, {0x68929a, 0x0, 0x1, {0x1a41d677, 0xedff07ec2, ...}, ...})\n\t/home/ubuntu/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1009 +0x8c9\ngithub.com/cometbft/cometbft/consensus.(*State).receiveRoutine(0xc0024f0388, 0x0)\n\t/home/ubuntu/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:891 +0x6bf\ncreated by github.com/cometbft/cometbft/consensus.(*State).OnStart in goroutine 434\n\t/home/ubuntu/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:411 +0x132\n"
```@evan-forbes: thanks, also helpful

> The RPC nodes seems to be fine:

makes sense, since this issue is with proposing blocks@rares-e2s: Our validator also stalled with this panic today. Had to restart it. 

```
`Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]: panic: writing square: square size 256 is too small to fit all blobs
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]: goroutine 382744 [running]:
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]: github.com/celestiaorg/celestia-app/v3/app.(*App).PrepareProposal(0xc000d55508, {0xc037e3bcc0, 0x7fffffff, {0xc010ba8888, 0x7}, 0x6890ee, {0x1b293f06, 0xedff07368, 0x0}})
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]:         /go/src/github.com/celestiaorg/celestia-app/v3/app/prepare_proposal.go:67 +0xbf6
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]: github.com/tendermint/tendermint/abci/types.(*GRPCApplication).PrepareProposal(0xc003a0cbd0, {0x27f6500?, 0xc002ac1900?}, 0xc037e3bc80)
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]:         /root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.55.0-tm-v0.34.35/abci/types/application.go:198 +0xc2
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]: github.com/tendermint/tendermint/abci/types._ABCIApplication_PrepareProposal_Handler({0x27f6500, 0xc003a0cbd0}, {0x336f648, 0xc0173f6570}, 0xc036420780, 0x0)
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]:         /root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.55.0-tm-v0.34.35/abci/types/types.pb.go:4379 +0x1a9
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]: google.golang.org/grpc.(*Server).processUnaryRPC(0xc003966a00, {0x336f648, 0xc02a213ce0}, {0x3382320, 0xc00085c680}, 0xc00373c6c0, 0xc00087da40, 0x49157a8, 0x0)
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]:         /root/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1394 +0xe2b
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]: google.golang.org/grpc.(*Server).handleStream(0xc003966a00, {0x3382320, 0xc00085c680}, 0xc00373c6c0)
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]:         /root/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1805 +0xe8b
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]: google.golang.org/grpc.(*Server).serveStreams.func2.1()
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]:         /root/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1029 +0x7f
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]: created by google.golang.org/grpc.(*Server).serveStreams.func2 in goroutine 159
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624994]:         /root/go/pkg/mod/google.golang.org/grpc@v1.68.0/server.go:1040 +0x125
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624888]: 2025/06/27 14:11:41 Process finished with error: exit status 2
Jun 27 14:11:41 rom247-sg-n4 celestia-appd[2624888]: 2:11PM ERR CONSENSUS FAILURE!!! err="rpc error: code = Unavailable desc = error reading from server: EOF" module=consensus stack="goroutine 1140 [running]:\nruntime/debug.Stack()\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.24.2.linux-amd64/src/runtime/debug/stack.go:26 +0x5e\ngithub.com/cometbft/cometbft/consensus.(*State).receiveRoutine.func2()\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:825 +0x46\npanic({0x402f0e0?, 0xc017087b70?})\n\t/root/go/pkg/mod/golang.org/toolchain@v0.0.1-go1.24.2.linux-amd64/src/runtime/panic.go:792 +0x132\ngithub.com/cometbft/cometbft/consensus.(*State).createProposalBlock(0xc002426e08, {0x4f4f140, 0xab936e0})\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1335 +0x28f\ngithub.com/cometbft/cometbft/consensus.(*State).defaultDecideProposal(0xc002426e08, 0x6890ee, 0x0)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1240 +0x65\ngithub.com/cometbft/cometbft/consensus.(*State).enterPropose(0xc002426e08, 0x6890ee, 0x0)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1219 +0x82e\ngithub.com/cometbft/cometbft/consensus.(*State).enterNewRound(0xc002426e08, 0x6890ee, 0x0)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1138 +0xb4f\ngithub.com/cometbft/cometbft/consensus.(*State).handleTimeout(0xc002426e08, {0x0?, 0x0?, 0x0?, 0x0?}, {0x6890ee, 0x0, 0x1, {0x11187bb, 0xedff0736d, ...}, ...})\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:1009 +0x8c9\ngithub.com/cometbft/cometbft/consensus.(*State).receiveRoutine(0xc002426e08, 0x0)\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:891 +0x6bf\ncreated by github.com/cometbft/cometbft/consensus.(*State).OnStart in goroutine 253\n\t/root/go/pkg/mod/github.com/celestiaorg/celestia-core@v1.54.1-tm-v0.38.17/consensus/state.go:411 +0x132\n"
```
`@evan-forbes: I've got a branch that has a special version of core that should save proposals in https://github.com/celestiaorg/celestia-app/tree/evan/save-proposals

its not tested, but I'm running it on mocha atm so if [my validator](https://mocha-4.celenium.io/validator/684049462?tab=Delegators) starts producing blocks that are bad it should save them and we can use those

others can feel free to run it as well if they want to help debug@evan-forbes: been running this for the past 24 hours https://github.com/celestiaorg/celestia-core/pull/1992#pullrequestreview-2968861593

still have yet to catch any invalid proposal blocks, but 🤞 @rares-e2s: Running your version @evan-forbes  https://github.com/celestiaorg/celestia-app/tree/evan/save-proposals 
Any specific logs we should look for?  From what I see, the node will restart itself, right? @rootulp: If you see the log `writing square: square size...` then please look in `{CELESTIA_APP_HOME}/data/debug` for a file that ends in `_faulty_proposal.json` or `_failed_proposal.pb`. If you have such a file, can you share it with us? Thanks!@rares-e2s: Thanks @rootulp, so far, no log and no debug  file. Will share it in case we get any

Solution patch:
diff --git a/app/validate_txs.go b/app/filtered_square_builder.go
similarity index 58%
rename from app/validate_txs.go
rename to app/filtered_square_builder.go
index 20d83556..b2562d75 100644
--- a/app/validate_txs.go
+++ b/app/filtered_square_builder.go
@@ -1,54 +1,65 @@
 package app
 
 import (
-	"cosmossdk.io/log"
 	tmbytes "github.com/cometbft/cometbft/libs/bytes"
 	coretypes "github.com/cometbft/cometbft/types"
 	"github.com/cosmos/cosmos-sdk/client"
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 
+	square "github.com/celestiaorg/go-square/v2"
 	"github.com/celestiaorg/go-square/v2/tx"
 
 	"github.com/celestiaorg/celestia-app/v4/pkg/appconsts"
 )
 
-// separateTxs decodes raw tendermint txs into normal and blob txs.
-func separateTxs(_ client.TxConfig, rawTxs [][]byte) ([][]byte, []*tx.BlobTx) {
-	normalTxs := make([][]byte, 0, len(rawTxs))
-	blobTxs := make([]*tx.BlobTx, 0, len(rawTxs))
-	for _, rawTx := range rawTxs {
-		bTx, isBlob, err := tx.UnmarshalBlobTx(rawTx)
-		if isBlob {
-			if err != nil {
-				panic(err)
-			}
-			blobTxs = append(blobTxs, bTx)
-		} else {
-			normalTxs = append(normalTxs, rawTx)
-		}
+// FilteredSquareBuilder filters txs and blobs using a copy of the state and tx validity
+// rules before adding it the square.
+type FilteredSquareBuilder struct {
+	handler  sdk.AnteHandler
+	txConfig client.TxConfig
+	builder  *square.Builder
+}
+
+func NewFilteredSquareBuilder(
+	handler sdk.AnteHandler,
+	txConfig client.TxConfig,
+	maxSquareSize,
+	subtreeRootThreshold int,
+) (*FilteredSquareBuilder, error) {
+	builder, err := square.NewBuilder(maxSquareSize, subtreeRootThreshold)
+	if err != nil {
+		return nil, err
 	}
-	return normalTxs, blobTxs
+	return &FilteredSquareBuilder{
+		handler:  handler,
+		txConfig: txConfig,
+		builder:  builder,
+	}, nil
+}
+
+func (fsb *FilteredSquareBuilder) Build() (square.Square, error) {
+	return fsb.builder.Export()
 }
 
-// FilterTxs applies the antehandler to all proposed transactions and removes
-// transactions that return an error.
-//
-// Side-effect: arranges all normal transactions before all blob transactions.
-func FilterTxs(logger log.Logger, ctx sdk.Context, handler sdk.AnteHandler, txConfig client.TxConfig, txs [][]byte) [][]byte {
-	normalTxs, blobTxs := separateTxs(txConfig, txs)
-	normalTxs, ctx = filterStdTxs(logger, txConfig.TxDecoder(), ctx, handler, normalTxs)
-	blobTxs, _ = filterBlobTxs(logger, txConfig.TxDecoder(), ctx, handler, blobTxs)
-	return append(normalTxs, encodeBlobTxs(blobTxs)...)
+func (fsb *FilteredSquareBuilder) Builder() *square.Builder {
+	return fsb.builder
 }
 
-// filterStdTxs applies the provided antehandler to each transaction and removes
-// transactions that return an error. Panics are caught by the checkTxValidity
-// function used to apply the ante handler.
-func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs [][]byte) ([][]byte, sdk.Context) {
-	n := 0
-	nonPFBMessageCount := 0
-	for _, tx := range txs {
+func (fsb *FilteredSquareBuilder) Fill(ctx sdk.Context, txs [][]byte) [][]byte {
+	logger := ctx.Logger().With("app/filtered-square-builder")
+
+	normalTxs, blobTxs := separateTxs(fsb.txConfig, txs)
+
+	var (
+		nonPFBMessageCount = 0
+		pfbMessageCount    = 0
+		dec                = fsb.txConfig.TxDecoder()
+		n                  = 0
+		m                  = 0
+	)
+
+	for _, tx := range normalTxs {
 		sdkTx, err := dec(tx)
 		if err != nil {
 			logger.Error("decoding already checked transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash()), "error", err)
@@ -63,9 +74,13 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler
 			logger.Debug("skipping tx because the max non PFB message count was reached", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash()))
 			continue
 		}
-		nonPFBMessageCount += len(sdkTx.GetMsgs())
 
-		ctx, err = handler(ctx, sdkTx, false)
+		if !fsb.builder.AppendTx(tx) {
+			logger.Debug("skipping tx because it was too large to fit in the square", "tx", tmbytes.HexBytes(coretypes.Tx(tx).Hash()))
+			continue
+		}
+
+		ctx, err = fsb.handler(ctx, sdkTx, false)
 		// either the transaction is invalid (ie incorrect nonce) and we
 		// simply want to remove this tx, or we're catching a panic from one
 		// of the anteHandlers which is logged.
@@ -77,23 +92,19 @@ func filterStdTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler
 				"msgs", msgTypes,
 			)
 			telemetry.IncrCounter(1, "prepare_proposal", "invalid_std_txs")
+			err = fsb.builder.RevertLastTx()
+			if err != nil {
+				logger.Error("reverting last transaction", "error", err)
+			}
 			continue
 		}
-		txs[n] = tx
-		n++
 
+		nonPFBMessageCount += len(sdkTx.GetMsgs())
+		normalTxs[n] = tx
+		n++
 	}
 
-	return txs[:n], ctx
-}
-
-// filterBlobTxs applies the provided antehandler to each transaction
-// and removes transactions that return an error. Panics are caught by the checkTxValidity
-// function used to apply the ante handler.
-func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handler sdk.AnteHandler, txs []*tx.BlobTx) ([]*tx.BlobTx, sdk.Context) {
-	n := 0
-	pfbMessageCount := 0
-	for _, tx := range txs {
+	for _, tx := range blobTxs {
 		sdkTx, err := dec(tx.Tx)
 		if err != nil {
 			logger.Error("decoding already checked blob transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash()), "error", err)
@@ -104,12 +115,16 @@ func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handle
 		ctx = ctx.WithTxBytes(tx.Tx)
 
 		if pfbMessageCount+len(sdkTx.GetMsgs()) > appconsts.MaxPFBMessages {
-			logger.Debug("skipping tx because the max pfb message count was reached", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash()))
+			logger.Debug("skipping blob tx because the max pfb message count was reached", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash()))
 			continue
 		}
-		pfbMessageCount += len(sdkTx.GetMsgs())
 
-		ctx, err = handler(ctx, sdkTx, false)
+		if !fsb.builder.AppendBlobTx(tx) {
+			logger.Debug("skipping tx because it was too large to fit in the square", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash()))
+			continue
+		}
+
+		ctx, err = fsb.handler(ctx, sdkTx, false)
 		// either the transaction is invalid (ie incorrect nonce) and we
 		// simply want to remove this tx, or we're catching a panic from one
 		// of the anteHandlers which is logged.
@@ -118,14 +133,22 @@ func filterBlobTxs(logger log.Logger, dec sdk.TxDecoder, ctx sdk.Context, handle
 				"filtering already checked blob transaction", "tx", tmbytes.HexBytes(coretypes.Tx(tx.Tx).Hash()), "error", err,
 			)
 			telemetry.IncrCounter(1, "prepare_proposal", "invalid_blob_txs")
+			err = fsb.builder.RevertLastBlobTx()
+			if err != nil {
+				logger.Error("reverting last blob transaction failed", "error", err)
+			}
 			continue
 		}
-		txs[n] = tx
-		n++
 
+		pfbMessageCount += len(sdkTx.GetMsgs())
+		blobTxs[m] = tx
+		m++
 	}
 
-	return txs[:n], ctx
+	kept := make([][]byte, 0, m+n)
+	kept = append(kept, normalTxs[:n]...)
+	kept = append(kept, encodeBlobTxs(blobTxs[:m])...)
+	return kept
 }
 
 func msgTypes(sdkTx sdk.Tx) []string {
@@ -148,3 +171,21 @@ func encodeBlobTxs(blobTxs []*tx.BlobTx) [][]byte {
 	}
 	return txs
 }
+
+// separateTxs decodes raw tendermint txs into normal and blob txs.
+func separateTxs(_ client.TxConfig, rawTxs [][]byte) ([][]byte, []*tx.BlobTx) {
+	normalTxs := make([][]byte, 0, len(rawTxs))
+	blobTxs := make([]*tx.BlobTx, 0, len(rawTxs))
+	for _, rawTx := range rawTxs {
+		bTx, isBlob, err := tx.UnmarshalBlobTx(rawTx)
+		if isBlob {
+			if err != nil {
+				panic(err)
+			}
+			blobTxs = append(blobTxs, bTx)
+		} else {
+			normalTxs = append(normalTxs, rawTx)
+		}
+	}
+	return normalTxs, blobTxs
+}
diff --git a/app/prepare_proposal.go b/app/prepare_proposal.go
index 0373db06..449b5058 100644
--- a/app/prepare_proposal.go
+++ b/app/prepare_proposal.go
@@ -7,7 +7,6 @@ import (
 	"github.com/cosmos/cosmos-sdk/telemetry"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 
-	"github.com/celestiaorg/go-square/v2"
 	"github.com/celestiaorg/go-square/v2/share"
 
 	"github.com/celestiaorg/celestia-app/v4/app/ante"
@@ -36,13 +35,20 @@ func (app *App) PrepareProposalHandler(ctx sdk.Context, req *abci.RequestPrepare
 		app.GovParamFilters(),
 	)
 
-	// Filter out invalid transactions.
-	txs := FilterTxs(app.Logger(), ctx, handler, app.encodingConfig.TxConfig, req.Txs)
+	fsb, err := NewFilteredSquareBuilder(
+		handler,
+		app.encodingConfig.TxConfig,
+		app.MaxEffectiveSquareSize(ctx),
+		appconsts.SubtreeRootThreshold,
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	txs := fsb.Fill(ctx, req.Txs)
 
 	// Build the square from the set of valid and prioritised transactions.
-	// The txs returned are the ones used in the square and block.
-	var dataSquare square.Square
-	dataSquare, txs, err := square.Build(txs, app.MaxEffectiveSquareSize(ctx), appconsts.SubtreeRootThreshold)
+	dataSquare, err := fsb.Build()
 	if err != nil {
 		panic(err)
 	}
diff --git a/go.mod b/go.mod
index 1720c1fd..e337d986 100644
--- a/go.mod
+++ b/go.mod
@@ -17,7 +17,7 @@ require (
 	cosmossdk.io/x/tx v0.13.8
 	cosmossdk.io/x/upgrade v0.1.4
 	github.com/bcp-innovations/hyperlane-cosmos v1.0.1
-	github.com/celestiaorg/go-square/v2 v2.2.0
+	github.com/celestiaorg/go-square/v2 v2.3.0
 	github.com/celestiaorg/knuu v0.16.3
 	github.com/celestiaorg/nmt v0.24.0
 	github.com/celestiaorg/rsmt2d v0.14.0
diff --git a/go.sum b/go.sum
index a5a14711..3a9293fa 100644
--- a/go.sum
+++ b/go.sum
@@ -360,8 +360,8 @@ github.com/celestiaorg/cosmos-sdk v1.29.2-sdk-v0.50.12 h1:Q5Y9mZymbQP9qUr/l5IPcg
 github.com/celestiaorg/cosmos-sdk v1.29.2-sdk-v0.50.12/go.mod h1:clLEg7hWK0iJfiO1Vp71tcQ0vGbFzVsEcLxxnRobIZM=
 github.com/celestiaorg/cosmos-sdk/x/upgrade v0.1.0 h1:+i3G5mP/kPgFEn83EEXGly29QDin2Gvdt0kgpmw/vTg=
 github.com/celestiaorg/cosmos-sdk/x/upgrade v0.1.0/go.mod h1:T4K9O18zQNKNpt4YvTL3lcUt4aKOEU05ZIFWVdQi3Ak=
-github.com/celestiaorg/go-square/v2 v2.2.0 h1:zJnUxCYc65S8FgUfVpyG/osDcsnjzo/JSXw/Uwn8zp4=
-github.com/celestiaorg/go-square/v2 v2.2.0/go.mod h1:j8kQUqJLYtcvCQMQV6QjEhUdaF7rBTXF74g8LbkR0Co=
+github.com/celestiaorg/go-square/v2 v2.3.0 h1:tVh6sZy1d2l5maVXUpc7eoTXdb3ptJVJt/U8z2XUWgQ=
+github.com/celestiaorg/go-square/v2 v2.3.0/go.mod h1:6M2txj0j6dkoE+cgwyG0EqrEPhbZpM2R1lsWEopMIBc=
 github.com/celestiaorg/knuu v0.16.3 h1:ORmcBoSW+67iPKF0yIGKwWMefphk5hvy08KklmCT0aw=
 github.com/celestiaorg/knuu v0.16.3/go.mod h1:nB7IGCR984YKEDW+j5xHPOidYfbO4DoZI1rDKijvF5E=
 github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc=



Test patch:
diff --git a/app/test/prepare_proposal_test.go b/app/test/prepare_proposal_test.go
index 358026d2..c10b7e35 100644
--- a/app/test/prepare_proposal_test.go
+++ b/app/test/prepare_proposal_test.go
@@ -7,6 +7,8 @@ import (
 	"time"
 
 	abci "github.com/cometbft/cometbft/abci/types"
+	cmtproto "github.com/cometbft/cometbft/proto/tendermint/types"
+	"github.com/cometbft/cometbft/proto/tendermint/version"
 	coretypes "github.com/cometbft/cometbft/types"
 	"github.com/cosmos/cosmos-sdk/crypto/hd"
 	"github.com/cosmos/cosmos-sdk/crypto/keyring"
@@ -30,6 +32,75 @@ import (
 	blobtypes "github.com/celestiaorg/celestia-app/v4/x/blob/types"
 )
 
+func TestPrepareProposalValidConstruction(t *testing.T) {
+	// Reproduces https://github.com/celestiaorg/celestia-app/issues/4961
+	t.Run("prepare proposal creates a proposal that process proposal accepts", func(t *testing.T) {
+		encConf := encoding.MakeConfig(app.ModuleEncodingRegisters...)
+		accounts := testfactory.GenerateAccounts(1)
+		testApp, kr := testutil.SetupTestAppWithGenesisValSetAndMaxSquareSize(app.DefaultConsensusParams(), 128, accounts...)
+		height := testApp.LastBlockHeight() + 1
+
+		txs := createBlobTxs(t, testApp, encConf, kr, accounts)
+		require.Equal(t, 9, len(txs))
+
+		prepareResponse, err := testApp.PrepareProposal(&abci.RequestPrepareProposal{
+			Txs:    txs,
+			Height: height,
+			Time:   time.Now(),
+		})
+		require.NoError(t, err)
+		// The filtered builder in prepare proposal should have dropped the last two txs.
+		// The filtered builder should have dropped the second to last tx because it was too large to fit in the square.
+		// The filtered builder should have dropped the last tx because the nonce for it was invalidated by dropping the second to last tx.
+		require.Equal(t, 7, len(prepareResponse.Txs))
+
+		processResponse, err := testApp.ProcessProposal(&abci.RequestProcessProposal{
+			Header: &cmtproto.Header{
+				Version: version.Consensus{
+					Block: 1,
+					App:   3,
+				},
+				ChainID:  testutil.ChainID,
+				Height:   height,
+				Time:     time.Now(),
+				DataHash: prepareResponse.DataRootHash,
+			},
+			Height:       height,
+			Txs:          prepareResponse.Txs,
+			SquareSize:   prepareResponse.SquareSize,
+			DataRootHash: prepareResponse.DataRootHash,
+		})
+
+		require.NoError(t, err)
+		require.NotNil(t, processResponse)
+		require.Equal(t, abci.ResponseProcessProposal_ACCEPT, processResponse.Status)
+	})
+}
+
+// createBlobTxs returns 9 blob transactions. The first 8 are 1 MiB each and the last one is 100 bytes.
+func createBlobTxs(t *testing.T, testApp *app.App, encConf encoding.Config, keyring keyring.Keyring, accounts []string) (txs [][]byte) {
+	accountName := accounts[0]
+	address := testfactory.GetAddress(keyring, accountName)
+	account := testutil.DirectQueryAccount(testApp, address)
+	sequence := account.GetSequence()
+	accountNumber := account.GetAccountNumber()
+
+	blobSize := 1 * mebibyte
+	blobCount := 1
+
+	for i := 0; i < 8; i++ {
+		tx := testutil.BlobTxWithManualSequence(t, encConf.TxConfig, keyring, blobSize, blobCount, testutil.ChainID, accountName, sequence, accountNumber)
+		txs = append(txs, tx)
+		sequence++
+	}
+
+	blobSize = 100 // bytes
+	tx := testutil.BlobTxWithManualSequence(t, encConf.TxConfig, keyring, blobSize, blobCount, testutil.ChainID, accountName, sequence, accountNumber)
+	txs = append(txs, tx)
+
+	return txs
+}
+
 func TestPrepareProposalPutsPFBsAtEnd(t *testing.T) {
 	numBlobTxs, numNormalTxs := 3, 3
 	accnts := testfactory.GenerateAccounts(numBlobTxs + numNormalTxs)
diff --git a/test/docker-e2e/go.mod b/test/docker-e2e/go.mod
index 91cc9209..d91b1295 100644
--- a/test/docker-e2e/go.mod
+++ b/test/docker-e2e/go.mod
@@ -4,7 +4,7 @@ go 1.23.6
 
 require (
 	github.com/celestiaorg/celestia-app/v4 v4.0.0
-	github.com/celestiaorg/go-square/v2 v2.2.0
+	github.com/celestiaorg/go-square/v2 v2.3.0
 	github.com/celestiaorg/tastora v0.0.0-20250515081551-795e795a49c7
 	github.com/cosmos/cosmos-sdk v0.50.13
 	github.com/docker/docker v28.2.2+incompatible
diff --git a/test/docker-e2e/go.sum b/test/docker-e2e/go.sum
index 665013d7..b615c772 100644
--- a/test/docker-e2e/go.sum
+++ b/test/docker-e2e/go.sum
@@ -350,8 +350,8 @@ github.com/celestiaorg/cosmos-sdk v1.29.2-sdk-v0.50.12 h1:Q5Y9mZymbQP9qUr/l5IPcg
 github.com/celestiaorg/cosmos-sdk v1.29.2-sdk-v0.50.12/go.mod h1:clLEg7hWK0iJfiO1Vp71tcQ0vGbFzVsEcLxxnRobIZM=
 github.com/celestiaorg/cosmos-sdk/x/upgrade v0.1.0 h1:+i3G5mP/kPgFEn83EEXGly29QDin2Gvdt0kgpmw/vTg=
 github.com/celestiaorg/cosmos-sdk/x/upgrade v0.1.0/go.mod h1:T4K9O18zQNKNpt4YvTL3lcUt4aKOEU05ZIFWVdQi3Ak=
-github.com/celestiaorg/go-square/v2 v2.2.0 h1:zJnUxCYc65S8FgUfVpyG/osDcsnjzo/JSXw/Uwn8zp4=
-github.com/celestiaorg/go-square/v2 v2.2.0/go.mod h1:j8kQUqJLYtcvCQMQV6QjEhUdaF7rBTXF74g8LbkR0Co=
+github.com/celestiaorg/go-square/v2 v2.3.0 h1:tVh6sZy1d2l5maVXUpc7eoTXdb3ptJVJt/U8z2XUWgQ=
+github.com/celestiaorg/go-square/v2 v2.3.0/go.mod h1:6M2txj0j6dkoE+cgwyG0EqrEPhbZpM2R1lsWEopMIBc=
 github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc=
 github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA=
 github.com/celestiaorg/nmt v0.24.0 h1:23u/mneledCG/6xWl1cZeTPrzRLpDTAHxMFsqoOL+B4=
diff --git a/test/util/direct_tx_gen.go b/test/util/direct_tx_gen.go
index c503e022..9732eb7d 100644
--- a/test/util/direct_tx_gen.go
+++ b/test/util/direct_tx_gen.go
@@ -234,3 +234,32 @@ func getAddress(account string, kr keyring.Keyring) sdk.AccAddress {
 	}
 	return addr
 }
+
+func BlobTxWithManualSequence(
+	t *testing.T,
+	cfg client.TxConfig,
+	kr keyring.Keyring,
+	blobSize int,
+	blobCount int,
+	chainid string,
+	account string,
+	sequence uint64,
+	accountNum uint64,
+) coretypes.Tx {
+	t.Helper()
+
+	opts := blobfactory.DefaultTxOpts()
+	addr := testfactory.GetAddress(kr, account)
+	acc := user.NewAccount(account, accountNum, sequence)
+	signer, err := user.NewSigner(kr, cfg, chainid, acc)
+	require.NoError(t, err)
+
+	msg, blobs := blobfactory.RandMsgPayForBlobsWithSigner(random.New(), addr.String(), blobSize, blobCount)
+	txBz, _, err := signer.CreateTx([]sdk.Msg{msg}, opts...)
+	require.NoError(t, err)
+
+	cTx, err := tx.MarshalBlobTx(txBz, blobs...)
+	require.NoError(t, err)
+
+	return cTx
+}
diff --git a/test/util/malicious/out_of_order_prepare.go b/test/util/malicious/out_of_order_prepare.go
index 6231a3f9..2060fe92 100644
--- a/test/util/malicious/out_of_order_prepare.go
+++ b/test/util/malicious/out_of_order_prepare.go
@@ -46,11 +46,21 @@ func (a *App) OutOfOrderPrepareProposal(req *abci.RequestPrepareProposal) (*abci
 		a.GovParamFilters(),
 	)
 
-	txs := app.FilterTxs(a.Logger(), sdkCtx, handler, a.GetTxConfig(), req.Txs)
+	fsb, err := app.NewFilteredSquareBuilder(
+		handler,
+		a.GetEncodingConfig().TxConfig,
+		a.MaxEffectiveSquareSize(sdkCtx),
+		appconsts.SubtreeRootThreshold,
+	)
+	if err != nil {
+		panic(err)
+	}
+
+	txs := fsb.Fill(sdkCtx, req.Txs)
 
 	// build the square from the set of valid and prioritised transactions.
 	// The txs returned are the ones used in the square and block
-	dataSquare, txs, err := Build(txs, appconsts.LatestVersion, a.MaxEffectiveSquareSize(sdkCtx), OutOfOrderExport)
+	dataSquare, err := OutOfOrderExport(fsb.Builder())
 	if err != nil {
 		panic(err)
 	}