Repository: ethereum/go-ethereum (language: go)

Issue:
Transactions stuck in pending in dev-mode
#### System information

* Geth version
    ```
    Geth
    Version: 1.13.14-stable
    Git Commit: 2bd6bd01d2e8561dd7fc21b631f4a34ac16627a1
    Git Commit Date: 20240227
    Architecture: arm64
    Go Version: go1.21.0
    Operating System: darwin
    GOPATH=
    GOROOT=/Users/gashkov/.asdf/installs/golang/1.22.0/go
    ```
* CL client & version: none (dev-mode)
* OS & Version: macOS Sonoma 14.4.1 (23E224)
* Commit hash: not applicable

#### Steps to reproduce the behaviour

1. Run `go-ethereum` in a dev-mode with block generation on demand
2. Send a batch of transactions concurrently

<details>
<summary>Stress-testing script I used for reproducing this behavior.</summary>

```go
package main

import (
	"fmt"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/common/hexutil"
	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/rpc"
	"log"
	"sync"
	"time"
)

func getCoinbase(c *rpc.Client) common.Address {
	var coinbase common.Address

	if err := c.Call(&coinbase, "eth_coinbase"); err != nil {
		log.Fatalln("error obtaining coinbase", err)
	} else {
		log.Printf("coinbase=%v", coinbase)
	}

	return coinbase
}

func getTransactionCount(c *rpc.Client, address common.Address) hexutil.Uint {
	var nonce hexutil.Uint

	if err := c.Call(&nonce, "eth_getTransactionCount", address, "latest"); err != nil {
		log.Fatalln("error obtaining coinbase's initial nonce", err)
	} else {
		log.Printf("nonce=%v", nonce)
	}

	return nonce
}

func sendTransactionToSelf(c *rpc.Client, address common.Address, nonce hexutil.Uint) (common.Hash, error) {
	txObject := map[string]any{"from": address, "to": address, "nonce": nonce}

	var txHash common.Hash
	if err := c.Call(&txHash, "eth_sendTransaction", txObject); err != nil {
		return common.Hash{}, err
	}

	return txHash, nil
}

func waitTransactionReceipt(c *rpc.Client, txHash common.Hash) (*types.Receipt, error) {
	for i := 0; i < 10; i++ {
		var txReceipt *types.Receipt
		if err := c.Call(&txReceipt, "eth_getTransactionReceipt", txHash); err != nil {
			if fmt.Sprint(err) == "transaction indexing is in progress" {
				// `transaction indexing is in progress` is returned when
				// `eth_getTransactionCount` is called with no previously mined
				// blocks.
				time.Sleep(1 * time.Second)
			} else {
				return nil, err
			}
		}

		if txReceipt != nil {
			return txReceipt, nil
		}

		time.Sleep(100 * time.Millisecond)
	}

	return nil, fmt.Errorf("timeout waiting tx hash=%v to be mined", txHash)
}

func main() {
	c, err := rpc.Dial("http://localhost:8545")

	if err != nil {
		log.Fatal("error dialing server", "url", "http://localhost:8545")
	}

	coinbase := getCoinbase(c)
	startingNonce := getTransactionCount(c, coinbase)

	var (
		wg  sync.WaitGroup
		sem = make(chan struct{}, 5)
	)

	for nonce := startingNonce; nonce <= startingNonce+15; nonce++ {
		nonce := nonce
		sem <- struct{}{}
		wg.Add(1)

		go func() {
			defer func() { <-sem }()
			defer wg.Done()

			if txHash, err := sendTransactionToSelf(c, coinbase, nonce); err != nil {
				log.Printf("error sending tx nonce=%v err=%v", uint64(nonce), err)
			} else if receipt, err := waitTransactionReceipt(c, txHash); err != nil {
				log.Printf("error waiting tx nonce=%v err=%v", uint64(nonce), err)
			} else {
				log.Printf("sent tx hash=%v nonce=%v blockNumber=%v", receipt.TxHash, uint64(nonce), receipt.BlockNumber)
			}
		}()
	}

	wg.Wait()
}
```
</details>
</details>
<details>
<summary>Stress-testing output</summary>

```
% go run main.go
2024/04/06 20:13:14 coinbase=0xfb8c69691E914275e19Eb9d42735844A120067aD
2024/04/06 20:13:14 nonce=0x0
2024/04/06 20:13:15 sent tx hash=0xad155b491b06bffa28d2e0ebd4cf4f85a8e5bec108428cb2492b0048395fc3b7 nonce=2 blockNumber=1
2024/04/06 20:13:15 sent tx hash=0x7c7d61049a6118838cb0babd50aa05d5f8feece647637313d2e432dc2448e6e4 nonce=1 blockNumber=1
2024/04/06 20:13:15 sent tx hash=0xa53f94890c86c1d20f6c6398c8cd07d8c91bad816c91b7119c525f597ae7237a nonce=0 blockNumber=1
2024/04/06 20:13:15 sent tx hash=0x06bbb3c89850442f73ac53fff35b42a529dac62dde0167480455dda21e4b54e4 nonce=4 blockNumber=1
2024/04/06 20:13:15 sent tx hash=0xb55986ad438a261b8225e6384eba83a3dd710f196a5105526f149d0c8f85c0a8 nonce=3 blockNumber=1
2024/04/06 20:13:15 sent tx hash=0xc2f5aa3063d679fd39d9f2495577cb204519e9b8ecd68065155083f9c13a9aae nonce=7 blockNumber=2
2024/04/06 20:13:15 sent tx hash=0xedb2c69e889019102540da5800c4717aa3e9178fa9f8a8e4ac260235827b122d nonce=5 blockNumber=2
2024/04/06 20:13:15 sent tx hash=0xcc36386a360e4c35ccf7da2cd6d87ec4509f619f6f1620a3a9e3c43d0a050f71 nonce=6 blockNumber=2
2024/04/06 20:13:15 sent tx hash=0xf7f8a37a85beb5d366c989295cd46961a9e2bb59fac54b58fcb69fd93c7caa25 nonce=8 blockNumber=2
2024/04/06 20:13:16 error waiting tx nonce=9 err=timeout waiting tx hash=0x9b54d66bb30c77334f95e2931ca4ab6f11f61b139af1a6fc517b3fb4fe8b5e93 to be mined
2024/04/06 20:13:16 error waiting tx nonce=11 err=timeout waiting tx hash=0x6bd2490d9e4539f37ef2493389cd1066a25b32b4b2f50076e2824437335ead8f to be mined
2024/04/06 20:13:16 error waiting tx nonce=13 err=timeout waiting tx hash=0x2298fea2beeae800aa07c9e1751de1a4995d3fc506e0af3fc9039d77e89309e9 to be mined
2024/04/06 20:13:16 error waiting tx nonce=10 err=timeout waiting tx hash=0x034b4941ac6c1de3fd8e32b5c9d9976fb5896c6dbd6f37e9bb019db9e136d825 to be mined
2024/04/06 20:13:16 error waiting tx nonce=12 err=timeout waiting tx hash=0x24ca4ec05d745c4271e7b4a70d7a74d403daa38306ef7a6f6ae01a8394db7964 to be mined
2024/04/06 20:13:17 error waiting tx nonce=14 err=timeout waiting tx hash=0x2120f1d967a28ba0c467fd6c638db89959d64a92872569e7e0869ba0f3abad43 to be mined
2024/04/06 20:13:17 error waiting tx nonce=15 err=timeout waiting tx hash=0xb0e0165e29eb44545cd852da79937ad50c5933bead69cf54fbc122b5b50163e6 to be mined
```

</details>

#### Expected behaviour

`geth --dev` accepts and executes transactions concurrently.

#### Actual behaviour

Transactions are accepted but stuck in the pool:

```
% curl \
  -s \
  -X POST http://localhost:8545/ \
  -H 'content-type: application/json' \
  -H 'accept: application/json, */*;q=0.5' \
  -d '{"jsonrpc":"2.0","id":"1","method":"txpool_inspect","params":[]}' | jq

{
  "jsonrpc": "2.0",
  "id": "1",
  "result": {
    "pending": {
      "0xfb8c69691E914275e19Eb9d42735844A120067aD": {
        "9": "0xfb8c69691E914275e19Eb9d42735844A120067aD: 0 wei + 21000 gas × 1750000001 wei"
      }
    },
    "queued": {
      "0xfb8c69691E914275e19Eb9d42735844A120067aD": {
        "10": "0xfb8c69691E914275e19Eb9d42735844A120067aD: 0 wei + 21000 gas × 1535240671 wei",
        "11": "0xfb8c69691E914275e19Eb9d42735844A120067aD: 0 wei + 21000 gas × 1535240671 wei",
        "12": "0xfb8c69691E914275e19Eb9d42735844A120067aD: 0 wei + 21000 gas × 1535240671 wei",
        "13": "0xfb8c69691E914275e19Eb9d42735844A120067aD: 0 wei + 21000 gas × 1535240671 wei",
        "14": "0xfb8c69691E914275e19Eb9d42735844A120067aD: 0 wei + 21000 gas × 1535240671 wei",
        "15": "0xfb8c69691E914275e19Eb9d42735844A120067aD: 0 wei + 21000 gas × 1535240671 wei"
      }
    }
  }
}
```

Looks like the root cause is the deadlock between `SimulatedBeacon` and
`TxPool` waiting each other. Overall, the following happens:

```
SimulatedBeacon.loop():
  case <-newTxs:
    SimulatedBeacon.SealBlock()
      TxPool.Sync()  // Runs pool reorg and waits until done
        LegacyPool.runReorg()
```

and at the same time

```
SubmitTransaction(newTx)
  EthAPIBackend.SendTx(newTx)
    LegacyPool.add(newTx)
      LegacyPool.runReorg()
        txFeed.send(newTx)
          newTxs <- newTx // Waits until SimulatedBeacon.loop will be ready
```

So, when new TXs are sent concurrently, `LegacyPool` waits `SimulatedBeacon`
to react to the new TX but the `SimulatedBeacon` itself waits `LegacyPool` to
sync itself.

#### Backtrace

Nothing too suspicious in logs, but stacktraces contain two goroutines
indirectly waiting each other though:

```
goroutine 4279 [chan receive, 1 minutes]:
github.com/ethereum/go-ethereum/core/txpool.(*TxPool).Sync(0xc000513e60)
        /Users/gashkov/dev/go-ethereum/core/txpool/txpool.go:478 +0x148
github.com/ethereum/go-ethereum/eth/catalyst.(*ConsensusAPI).forkchoiceUpdated(0xc000178be0, {{0xc9, 0x10, 0x26, 0x77, 0xa1, 0x3, 0x24, 0x7, 0xab, ...}, ...}, ...)
        /Users/gashkov/dev/go-ethereum/eth/catalyst/api.go:397 +0x28d0
github.com/ethereum/go-ethereum/eth/catalyst.(*SimulatedBeacon).sealBlock(0xc0006626e0, {0x108c0da00, 0x0, 0x0}, 0x661190bc)
        /Users/gashkov/dev/go-ethereum/eth/catalyst/simulated_beacon.go:159 +0x4f4
github.com/ethereum/go-ethereum/eth/catalyst.(*SimulatedBeacon).Commit(0xc0006626e0)
        /Users/gashkov/dev/go-ethereum/eth/catalyst/simulated_beacon.go:249 +0xc4
github.com/ethereum/go-ethereum/eth/catalyst.(*api).loop(0xc0001b6980)
        /Users/gashkov/dev/go-ethereum/eth/catalyst/simulated_beacon_api.go:50 +0x2d8
created by github.com/ethereum/go-ethereum/eth/catalyst.RegisterSimulatedBeaconAPIs in goroutine 1
        /Users/gashkov/dev/go-ethereum/eth/catalyst/simulated_beacon.go:294 +0x14c
```
```
goroutine 4369 [select, 1 minutes]:
reflect.rselect({0xc00001d148, 0x2, 0x99?})
        /Users/gashkov/.asdf/installs/golang/1.21.0/go/src/runtime/select.go:589 +0x2d0
reflect.Select({0xc000000b40, 0x2, 0x5})
        /Users/gashkov/.asdf/installs/golang/1.21.0/go/src/reflect/value.go:3104 +0xa00
github.com/ethereum/go-ethereum/event.(*Feed).Send(0xc0003b9c90, {0x106d49200, 0xc000b78078})
        /Users/gashkov/dev/go-ethereum/event/feed.go:160 +0x758
github.com/ethereum/go-ethereum/core/txpool/legacypool.(*LegacyPool).runReorg(0xc0003b9c00, 0xc0005e8fc0, 0xc000488ed0, 0xc000458ce0, 0xc00089d290)
        /Users/gashkov/dev/go-ethereum/core/txpool/legacypool/legacypool.go:1336 +0xbf0
created by github.com/ethereum/go-ethereum/core/txpool/legacypool.(*LegacyPool).scheduleReorgLoop in goroutine 46
        /Users/gashkov/dev/go-ethereum/core/txpool/legacypool/legacypool.go:1205 +0x37c
```

* [All logs](https://raw.githubusercontent.com/nikitagashkov/go-ethereum-1-13-4-dev-mode-deadlock-report/70ef96998d9d3a1b37dd6f1065b20b539a84a122/logs.txt)
* [All stacktraces](https://raw.githubusercontent.com/nikitagashkov/go-ethereum-1-13-4-dev-mode-deadlock-report/70ef96998d9d3a1b37dd6f1065b20b539a84a122/stacktrace.txt)


Thank you for reviewing this report!


Discussion / hints:
:sparkles: Thanks to the community, there's an estimated bounty value of **$10.90 USD** for a successful merge request of this issue via contributions such as **0.975 UNI** tokens. Happy coding :grinning:
Details and T&Cs at [joinfuel.io](https://joinfuel.io)
PSA: @fuelmessenger is not anything we're familiar with , might be a scam, so I'll hide those posts
I'm experiencing similar behavior, using Web3.js `1.x` and connecting through `IPC`. 
Ok not clear I it's the same problem, my script get stuck even when sending one transaction. I tried the referenced PR also and didn't change anything on my side. Going to check that further.
Ok it was not related after all. The stuckness I am experiencing was done first sequentially, one transaction after the other and not in parallel.

- Web3.js 1.3.0 is doing a `eth_getTransactionReceipt` call but was getting `WARN [04-09|16:08:34.719] Served eth_getTransactionReceipt         reqid=1368187869315370 duration="36.333µs" err="transaction indexing is in progress" errdata="transaction indexing is in progress"`
- This version of Web3.js I had was never retrying correctly leading to being stuck. Updating to Web3.js 1.10 fixed the issue. It's still stalls for a bit but it resolves at some point.

I have problem too sending parallel transactions to `geth --dev` using `Web3.js` hitting `already known` error through IPC, but I'll dig further here to understand it better and maybe I'll open a different issue.
 simaoueh's solution does not solve my problem
I use web3py
I think the problem is the upgrade of geth in dev mode
Because there was none in version 1.13.5
> Ok it was not related after all. The stuckness I am experiencing was done first sequentially, one transaction after the other and not in parallel.
> 
>     * Web3.js 1.3.0 is doing a `eth_getTransactionReceipt` call but was getting `WARN [04-09|16:08:34.719] Served eth_getTransactionReceipt         reqid=1368187869315370 duration="36.333µs" err="transaction indexing is in progress" errdata="transaction indexing is in progress"`
> 
>     * This version of Web3.js I had was never retrying correctly leading to being stuck. Updating to Web3.js 1.10 fixed the issue. It's still stalls for a bit but it resolves at some point.
> 
> 
> I have problem too sending parallel transactions to `geth --dev` using `Web3.js` hitting `already known` error through IPC, but I'll dig further here to understand it better and maybe I'll open a different issue.

The reason is that the transaction indexing is in progress error
What does it give?

Fix patch:
diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go
index 00cce259c861..36a332d92264 100644
--- a/eth/catalyst/api.go
+++ b/eth/catalyst/api.go
@@ -184,7 +184,7 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update engine.ForkchoiceStateV1, pa
 			return engine.STATUS_INVALID, engine.InvalidParams.With(errors.New("forkChoiceUpdateV1 called post-shanghai"))
 		}
 	}
-	return api.forkchoiceUpdated(update, payloadAttributes, engine.PayloadV1, false)
+	return api.forkchoiceUpdated(update, payloadAttributes, engine.PayloadV1)
 }
 
 // ForkchoiceUpdatedV2 is equivalent to V1 with the addition of withdrawals in the payload
@@ -207,7 +207,7 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV2(update engine.ForkchoiceStateV1, pa
 			return engine.STATUS_INVALID, engine.UnsupportedFork.With(errors.New("forkchoiceUpdatedV2 must only be called with paris and shanghai payloads"))
 		}
 	}
-	return api.forkchoiceUpdated(update, params, engine.PayloadV2, false)
+	return api.forkchoiceUpdated(update, params, engine.PayloadV2)
 }
 
 // ForkchoiceUpdatedV3 is equivalent to V2 with the addition of parent beacon block root
@@ -228,10 +228,10 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV3(update engine.ForkchoiceStateV1, pa
 	// hash, even if params are wrong. To do this we need to split up
 	// forkchoiceUpdate into a function that only updates the head and then a
 	// function that kicks off block construction.
-	return api.forkchoiceUpdated(update, params, engine.PayloadV3, false)
+	return api.forkchoiceUpdated(update, params, engine.PayloadV3)
 }
 
-func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes, payloadVersion engine.PayloadVersion, simulatorMode bool) (engine.ForkChoiceResponse, error) {
+func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payloadAttributes *engine.PayloadAttributes, payloadVersion engine.PayloadVersion) (engine.ForkChoiceResponse, error) {
 	api.forkchoiceLock.Lock()
 	defer api.forkchoiceLock.Unlock()
 
@@ -374,19 +374,6 @@ func (api *ConsensusAPI) forkchoiceUpdated(update engine.ForkchoiceStateV1, payl
 		if api.localBlocks.has(id) {
 			return valid(&id), nil
 		}
-		// If the beacon chain is ran by a simulator, then transaction insertion,
-		// block insertion and block production will happen without any timing
-		// delay between them. This will cause flaky simulator executions due to
-		// the transaction pool running its internal reset operation on a back-
-		// ground thread. To avoid the racey behavior - in simulator mode - the
-		// pool will be explicitly blocked on its reset before continuing to the
-		// block production below.
-		if simulatorMode {
-			if err := api.eth.TxPool().Sync(); err != nil {
-				log.Error("Failed to sync transaction pool", "err", err)
-				return valid(nil), engine.InvalidPayloadAttributes.With(err)
-			}
-		}
 		payload, err := api.eth.Miner().BuildPayload(args)
 		if err != nil {
 			log.Error("Failed to build payload", "err", err)
diff --git a/eth/catalyst/simulated_beacon.go b/eth/catalyst/simulated_beacon.go
index 8bdf94b80e81..86355a153387 100644
--- a/eth/catalyst/simulated_beacon.go
+++ b/eth/catalyst/simulated_beacon.go
@@ -20,6 +20,7 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"errors"
+	"fmt"
 	"math/big"
 	"sync"
 	"time"
@@ -30,6 +31,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto/kzg4844"
 	"github.com/ethereum/go-ethereum/eth"
+	"github.com/ethereum/go-ethereum/event"
 	"github.com/ethereum/go-ethereum/log"
 	"github.com/ethereum/go-ethereum/node"
 	"github.com/ethereum/go-ethereum/params"
@@ -41,36 +43,46 @@ const devEpochLength = 32
 // withdrawalQueue implements a FIFO queue which holds withdrawals that are
 // pending inclusion.
 type withdrawalQueue struct {
-	pending chan *types.Withdrawal
+	pending types.Withdrawals
+	mu      sync.Mutex
+	feed    event.Feed
+	subs    event.SubscriptionScope
 }
 
+type newWithdrawalsEvent struct{ Withdrawals types.Withdrawals }
+
 // add queues a withdrawal for future inclusion.
 func (w *withdrawalQueue) add(withdrawal *types.Withdrawal) error {
-	select {
-	case w.pending <- withdrawal:
-		break
-	default:
-		return errors.New("withdrawal queue full")
-	}
+	w.mu.Lock()
+	w.pending = append(w.pending, withdrawal)
+	w.mu.Unlock()
+
+	w.feed.Send(newWithdrawalsEvent{types.Withdrawals{withdrawal}})
 	return nil
 }
 
-// gatherPending returns a number of queued withdrawals up to a maximum count.
-func (w *withdrawalQueue) gatherPending(maxCount int) []*types.Withdrawal {
-	withdrawals := []*types.Withdrawal{}
-	for {
-		select {
-		case withdrawal := <-w.pending:
-			withdrawals = append(withdrawals, withdrawal)
-			if len(withdrawals) == maxCount {
-				return withdrawals
-			}
-		default:
-			return withdrawals
-		}
-	}
+// pop dequeues the specified number of withdrawals from the queue.
+func (w *withdrawalQueue) pop(count int) types.Withdrawals {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	count = min(count, len(w.pending))
+	popped := w.pending[0:count]
+	w.pending = w.pending[count:]
+
+	return popped
+}
+
+// subscribe allows a listener to be updated when new withdrawals are added to
+// the queue.
+func (w *withdrawalQueue) subscribe(ch chan<- newWithdrawalsEvent) event.Subscription {
+	sub := w.feed.Subscribe(ch)
+	return w.subs.Track(sub)
 }
 
+// SimulatedBeacon drives an Ethereum instance as if it were a real beacon
+// client. It can run in period mode where it mines a new block every period
+// (seconds) or on every transaction via Commit, Fork and AdjustTime.
 type SimulatedBeacon struct {
 	shutdownCh  chan struct{}
 	eth         *eth.Ethereum
@@ -86,10 +98,6 @@ type SimulatedBeacon struct {
 }
 
 // NewSimulatedBeacon constructs a new simulated beacon chain.
-// Period sets the period in which blocks should be produced.
-//
-//   - If period is set to 0, a block is produced on every transaction.
-//     via Commit, Fork and AdjustTime.
 func NewSimulatedBeacon(period uint64, eth *eth.Ethereum) (*SimulatedBeacon, error) {
 	block := eth.BlockChain().CurrentBlock()
 	current := engine.ForkchoiceStateV1{
@@ -112,7 +120,6 @@ func NewSimulatedBeacon(period uint64, eth *eth.Ethereum) (*SimulatedBeacon, err
 		engineAPI:          engineAPI,
 		lastBlockTime:      block.Time,
 		curForkchoiceState: current,
-		withdrawals:        withdrawalQueue{make(chan *types.Withdrawal, 20)},
 	}, nil
 }
 
@@ -156,6 +163,16 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u
 		c.setCurrentState(header.Hash(), *finalizedHash)
 	}
 
+	// Because transaction insertion, block insertion, and block production will
+	// happen without any timing delay between them in simulator mode and the
+	// transaction pool will be running its internal reset operation on a
+	// background thread, flaky executions can happen. To avoid the racey
+	// behavior, the pool will be explicitly blocked on its reset before
+	// continuing to the block production below.
+	if err := c.eth.APIBackend.TxPool().Sync(); err != nil {
+		return fmt.Errorf("failed to sync txpool: %w", err)
+	}
+
 	var random [32]byte
 	rand.Read(random[:])
 	fcResponse, err := c.engineAPI.forkchoiceUpdated(c.curForkchoiceState, &engine.PayloadAttributes{
@@ -164,13 +181,14 @@ func (c *SimulatedBeacon) sealBlock(withdrawals []*types.Withdrawal, timestamp u
 		Withdrawals:           withdrawals,
 		Random:                random,
 		BeaconRoot:            &common.Hash{},
-	}, engine.PayloadV3, true)
+	}, engine.PayloadV3)
 	if err != nil {
 		return err
 	}
 	if fcResponse == engine.STATUS_SYNCING {
 		return errors.New("chain rewind prevented invocation of payload creation")
 	}
+
 	envelope, err := c.engineAPI.getPayload(*fcResponse.PayloadID, true)
 	if err != nil {
 		return err
@@ -223,8 +241,7 @@ func (c *SimulatedBeacon) loop() {
 		case <-c.shutdownCh:
 			return
 		case <-timer.C:
-			withdrawals := c.withdrawals.gatherPending(10)
-			if err := c.sealBlock(withdrawals, uint64(time.Now().Unix())); err != nil {
+			if err := c.sealBlock(c.withdrawals.pop(10), uint64(time.Now().Unix())); err != nil {
 				log.Warn("Error performing sealing work", "err", err)
 			} else {
 				timer.Reset(time.Second * time.Duration(c.period))
@@ -260,7 +277,7 @@ func (c *SimulatedBeacon) setCurrentState(headHash, finalizedHash common.Hash) {
 
 // Commit seals a block on demand.
 func (c *SimulatedBeacon) Commit() common.Hash {
-	withdrawals := c.withdrawals.gatherPending(10)
+	withdrawals := c.withdrawals.pop(10)
 	if err := c.sealBlock(withdrawals, uint64(time.Now().Unix())); err != nil {
 		log.Warn("Error performing sealing work", "err", err)
 	}
@@ -301,16 +318,14 @@ func (c *SimulatedBeacon) AdjustTime(adjustment time.Duration) error {
 	if parent == nil {
 		return errors.New("parent not found")
 	}
-	withdrawals := c.withdrawals.gatherPending(10)
+	withdrawals := c.withdrawals.pop(10)
 	return c.sealBlock(withdrawals, parent.Time+uint64(adjustment/time.Second))
 }
 
+// RegisterSimulatedBeaconAPIs registers the simulated beacon's API with the
+// stack.
 func RegisterSimulatedBeaconAPIs(stack *node.Node, sim *SimulatedBeacon) {
-	api := &api{sim}
-	if sim.period == 0 {
-		// mine on demand if period is set to 0
-		go api.loop()
-	}
+	api := newSimulatedBeaconAPI(sim)
 	stack.RegisterAPIs([]rpc.API{
 		{
 			Namespace: "dev",
diff --git a/eth/catalyst/simulated_beacon_api.go b/eth/catalyst/simulated_beacon_api.go
index 73d0a5921d83..668780531501 100644
--- a/eth/catalyst/simulated_beacon_api.go
+++ b/eth/catalyst/simulated_beacon_api.go
@@ -18,44 +18,88 @@ package catalyst
 
 import (
 	"context"
-	"time"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core"
 	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ethereum/go-ethereum/log"
 )
 
-type api struct {
+// simulatedBeaconAPI provides a RPC API for SimulatedBeacon.
+type simulatedBeaconAPI struct {
 	sim *SimulatedBeacon
 }
 
-func (a *api) loop() {
+// newSimulatedBeaconAPI returns an instance of simulatedBeaconAPI with a
+// buffered commit channel. If period is zero, it starts a goroutine to handle
+// new tx events.
+func newSimulatedBeaconAPI(sim *SimulatedBeacon) *simulatedBeaconAPI {
+	api := &simulatedBeaconAPI{sim: sim}
+	if sim.period == 0 {
+		// mine on demand if period is set to 0
+		go api.loop()
+	}
+	return api
+}
+
+// loop is the main loop for the API when it's running in period = 0 mode. It
+// ensures that block production is triggered as soon as a new withdrawal or
+// transaction is received.
+func (a *simulatedBeaconAPI) loop() {
 	var (
-		newTxs = make(chan core.NewTxsEvent)
-		sub    = a.sim.eth.TxPool().SubscribeTransactions(newTxs, true)
+		newTxs    = make(chan core.NewTxsEvent)
+		newWxs    = make(chan newWithdrawalsEvent)
+		newTxsSub = a.sim.eth.TxPool().SubscribeTransactions(newTxs, true)
+		newWxsSub = a.sim.withdrawals.subscribe(newWxs)
+		doCommit  = make(chan struct{}, 1)
 	)
-	defer sub.Unsubscribe()
+	defer newTxsSub.Unsubscribe()
+	defer newWxsSub.Unsubscribe()
+
+	// A background thread which signals to the simulator when to commit
+	// based on messages over doCommit.
+	go func() {
+		for range doCommit {
+			a.sim.Commit()
+			a.sim.eth.TxPool().Sync()
+
+			// It's worth noting that in case a tx ends up in the pool listed as
+			// "executable", but for whatever reason the miner does not include it in
+			// a block -- maybe the miner is enforcing a higher tip than the pool --
+			// this code will spinloop.
+			for {
+				if executable, _ := a.sim.eth.TxPool().Stats(); executable == 0 {
+					break
+				}
+				a.sim.Commit()
+			}
+		}
+	}()
 
 	for {
 		select {
 		case <-a.sim.shutdownCh:
+			close(doCommit)
 			return
-		case w := <-a.sim.withdrawals.pending:
-			withdrawals := append(a.sim.withdrawals.gatherPending(9), w)
-			if err := a.sim.sealBlock(withdrawals, uint64(time.Now().Unix())); err != nil {
-				log.Warn("Error performing sealing work", "err", err)
+		case <-newWxs:
+			select {
+			case doCommit <- struct{}{}:
+			default:
 			}
 		case <-newTxs:
-			a.sim.Commit()
+			select {
+			case doCommit <- struct{}{}:
+			default:
+			}
 		}
 	}
 }
 
-func (a *api) AddWithdrawal(ctx context.Context, withdrawal *types.Withdrawal) error {
+// AddWithdrawal adds a withdrawal to the pending queue.
+func (a *simulatedBeaconAPI) AddWithdrawal(ctx context.Context, withdrawal *types.Withdrawal) error {
 	return a.sim.withdrawals.add(withdrawal)
 }
 
-func (a *api) SetFeeRecipient(ctx context.Context, feeRecipient common.Address) {
+// SetFeeRecipient sets the fee recipient for block building purposes.
+func (a *simulatedBeaconAPI) SetFeeRecipient(ctx context.Context, feeRecipient common.Address) {
 	a.sim.setFeeRecipient(feeRecipient)
 }


Test patch:
diff --git a/eth/catalyst/simulated_beacon_test.go b/eth/catalyst/simulated_beacon_test.go
index bb10938c359d..711e8f1d60f3 100644
--- a/eth/catalyst/simulated_beacon_test.go
+++ b/eth/catalyst/simulated_beacon_test.go
@@ -35,7 +35,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 )
 
-func startSimulatedBeaconEthService(t *testing.T, genesis *core.Genesis) (*node.Node, *eth.Ethereum, *SimulatedBeacon) {
+func startSimulatedBeaconEthService(t *testing.T, genesis *core.Genesis, period uint64) (*node.Node, *eth.Ethereum, *SimulatedBeacon) {
 	t.Helper()
 
 	n, err := node.New(&node.Config{
@@ -55,7 +55,7 @@ func startSimulatedBeaconEthService(t *testing.T, genesis *core.Genesis) (*node.
 		t.Fatal("can't create eth service:", err)
 	}
 
-	simBeacon, err := NewSimulatedBeacon(1, ethservice)
+	simBeacon, err := NewSimulatedBeacon(period, ethservice)
 	if err != nil {
 		t.Fatal("can't create simulated beacon:", err)
 	}
@@ -87,7 +87,7 @@ func TestSimulatedBeaconSendWithdrawals(t *testing.T) {
 	// short period (1 second) for testing purposes
 	var gasLimit uint64 = 10_000_000
 	genesis := core.DeveloperGenesisBlock(gasLimit, &testAddr)
-	node, ethService, mock := startSimulatedBeaconEthService(t, genesis)
+	node, ethService, mock := startSimulatedBeaconEthService(t, genesis, 1)
 	_ = mock
 	defer node.Close()
 
@@ -140,3 +140,65 @@ func TestSimulatedBeaconSendWithdrawals(t *testing.T) {
 		}
 	}
 }
+
+// Tests that zero-period dev mode can handle a lot of simultaneous
+// transactions/withdrawals
+func TestOnDemandSpam(t *testing.T) {
+	var (
+		withdrawals     []types.Withdrawal
+		txs                    = make(map[common.Hash]*types.Transaction)
+		testKey, _             = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
+		testAddr               = crypto.PubkeyToAddress(testKey.PublicKey)
+		gasLimit        uint64 = 10_000_000
+		genesis                = core.DeveloperGenesisBlock(gasLimit, &testAddr)
+		node, eth, mock        = startSimulatedBeaconEthService(t, genesis, 0)
+		_                      = newSimulatedBeaconAPI(mock)
+		signer                 = types.LatestSigner(eth.BlockChain().Config())
+		chainHeadCh            = make(chan core.ChainHeadEvent, 100)
+		sub                    = eth.BlockChain().SubscribeChainHeadEvent(chainHeadCh)
+	)
+	defer node.Close()
+	defer sub.Unsubscribe()
+
+	// generate some withdrawals
+	for i := 0; i < 20; i++ {
+		withdrawals = append(withdrawals, types.Withdrawal{Index: uint64(i)})
+		if err := mock.withdrawals.add(&withdrawals[i]); err != nil {
+			t.Fatal("addWithdrawal failed", err)
+		}
+	}
+
+	// generate a bunch of transactions
+	for i := 0; i < 20000; i++ {
+		tx, err := types.SignTx(types.NewTransaction(uint64(i), common.Address{byte(i), byte(1)}, big.NewInt(1000), params.TxGas, big.NewInt(params.InitialBaseFee*2), nil), signer, testKey)
+		if err != nil {
+			t.Fatal("error signing transaction", err)
+		}
+		txs[tx.Hash()] = tx
+		if err := eth.APIBackend.SendTx(context.Background(), tx); err != nil {
+			t.Fatal("error adding txs to pool", err)
+		}
+	}
+
+	var (
+		includedTxs = make(map[common.Hash]struct{})
+		includedWxs []uint64
+	)
+	for {
+		select {
+		case evt := <-chainHeadCh:
+			for _, itx := range evt.Block.Transactions() {
+				includedTxs[itx.Hash()] = struct{}{}
+			}
+			for _, iwx := range evt.Block.Withdrawals() {
+				includedWxs = append(includedWxs, iwx.Index)
+			}
+			// ensure all withdrawals/txs included. this will take two blocks b/c number of withdrawals > 10
+			if len(includedTxs) == len(txs) && len(includedWxs) == len(withdrawals) {
+				return
+			}
+		case <-time.After(10 * time.Second):
+			t.Fatalf("timed out without including all withdrawals/txs: have txs %d, want %d, have wxs %d, want %d", len(includedTxs), len(txs), len(includedWxs), len(withdrawals))
+		}
+	}
+}