Repository: tetratelabs/wazero
Pull request #2089 (2024-02-24 01:17:40)

wazevo(ssa): refactors pass execution
no functional change here


Patch:
diff --git a/internal/engine/wazevo/engine.go b/internal/engine/wazevo/engine.go
index 131a47bc..d654c3ab 100644
--- a/internal/engine/wazevo/engine.go
+++ b/internal/engine/wazevo/engine.go
@@ -351,17 +351,6 @@ func (e *engine) compileLocalWasmFunction(
 		wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "Optimized SSA", ssaBuilder.Format())
 	}
 
-	// Finalize the layout of SSA blocks which might use the optimization results.
-	ssaBuilder.LayoutBlocks()
-
-	if wazevoapi.PrintBlockLaidOutSSA && wazevoapi.PrintEnabledIndex(ctx) {
-		fmt.Printf("[[[Laidout SSA for %s]]]%s\n", wazevoapi.GetCurrentFunctionName(ctx), ssaBuilder.Format())
-	}
-
-	if wazevoapi.DeterministicCompilationVerifierEnabled {
-		wazevoapi.VerifyOrSetDeterministicCompilationContextValue(ctx, "Block laid out SSA", ssaBuilder.Format())
-	}
-
 	// Now our ssaBuilder contains the necessary information to further lower them to
 	// machine code.
 	original, rels, err := be.Compile(ctx)
diff --git a/internal/engine/wazevo/ssa/builder.go b/internal/engine/wazevo/ssa/builder.go
index b89c9e7e..34368ac6 100644
--- a/internal/engine/wazevo/ssa/builder.go
+++ b/internal/engine/wazevo/ssa/builder.go
@@ -98,13 +98,6 @@ type Builder interface {
 	// The returned slice must not be modified.
 	ValueRefCounts() []int
 
-	// LayoutBlocks layouts the BasicBlock(s) so that backend can easily generate the code.
-	// During its process, it splits the critical edges in the function.
-	// This must be called after RunPasses. Otherwise, it panics.
-	//
-	// The resulting order is available via BlockIteratorReversePostOrderBegin and BlockIteratorReversePostOrderNext.
-	LayoutBlocks()
-
 	// BlockIteratorReversePostOrderBegin is almost the same as BlockIteratorBegin except it returns the BasicBlock in the reverse post-order.
 	// This is available after RunPasses is run.
 	BlockIteratorReversePostOrderBegin() BasicBlock
@@ -196,10 +189,12 @@ type builder struct {
 	// blockIterCur is used to implement blockIteratorBegin and blockIteratorNext.
 	blockIterCur int
 
-	// donePasses is true if RunPasses is called.
-	donePasses bool
+	// donePreBlockLayoutPasses is true if all the passes before LayoutBlocks are called.
+	donePreBlockLayoutPasses bool
 	// doneBlockLayout is true if LayoutBlocks is called.
 	doneBlockLayout bool
+	// donePostBlockLayoutPasses is true if all the passes after LayoutBlocks are called.
+	donePostBlockLayoutPasses bool
 
 	currentSourceOffset SourceOffset
 }
@@ -221,7 +216,9 @@ func (b *builder) Init(s *Signature) {
 	b.instructionsPool.Reset()
 	b.basicBlocksPool.Reset()
 	b.varLengthPool.Reset()
-	b.donePasses = false
+	b.donePreBlockLayoutPasses = false
+	b.doneBlockLayout = false
+	b.donePostBlockLayoutPasses = false
 	for _, sig := range b.signatures {
 		sig.used = false
 	}
@@ -246,7 +243,6 @@ func (b *builder) Init(s *Signature) {
 	}
 	b.nextValueID = 0
 	b.reversePostOrderedBasicBlocks = b.reversePostOrderedBasicBlocks[:0]
-	b.donePasses = false
 	b.doneBlockLayout = false
 	for i := range b.valueRefCounts {
 		b.valueRefCounts[i] = 0
@@ -709,345 +705,6 @@ func (b *builder) BlockIDMax() BasicBlockID {
 	return BasicBlockID(b.basicBlocksPool.Allocated())
 }
 
-// LayoutBlocks implements Builder.LayoutBlocks. This re-organizes builder.reversePostOrderedBasicBlocks.
-//
-// TODO: there are tons of room for improvement here. e.g. LLVM has BlockPlacementPass using BlockFrequencyInfo,
-// BranchProbabilityInfo, and LoopInfo to do a much better job. Also, if we have the profiling instrumentation
-// like ball-larus algorithm, then we could do profile-guided optimization. Basically all of them are trying
-// to maximize the fall-through opportunities which is most efficient.
-//
-// Here, fallthrough happens when a block ends with jump instruction whose target is the right next block in the
-// builder.reversePostOrderedBasicBlocks.
-//
-// Currently, we just place blocks using the DFS reverse post-order of the dominator tree with the heuristics:
-//  1. a split edge trampoline towards a loop header will be placed as a fallthrough.
-//  2. we invert the brz and brnz if it makes the fallthrough more likely.
-//
-// This heuristic is done in maybeInvertBranches function.
-func (b *builder) LayoutBlocks() {
-	if !b.donePasses {
-		panic("LayoutBlocks must be called after all passes are done")
-	}
-
-	b.clearBlkVisited()
-
-	// We might end up splitting critical edges which adds more basic blocks,
-	// so we store the currently existing basic blocks in nonSplitBlocks temporarily.
-	// That way we can iterate over the original basic blocks while appending new ones into reversePostOrderedBasicBlocks.
-	nonSplitBlocks := b.blkStack[:0]
-	for i, blk := range b.reversePostOrderedBasicBlocks {
-		if !blk.Valid() {
-			continue
-		}
-		nonSplitBlocks = append(nonSplitBlocks, blk)
-		if i != len(b.reversePostOrderedBasicBlocks)-1 {
-			_ = maybeInvertBranches(blk, b.reversePostOrderedBasicBlocks[i+1])
-		}
-	}
-
-	var trampolines []*basicBlock
-
-	// Reset the order slice since we update on the fly by splitting critical edges.
-	b.reversePostOrderedBasicBlocks = b.reversePostOrderedBasicBlocks[:0]
-	uninsertedTrampolines := b.blkStack2[:0]
-	for _, blk := range nonSplitBlocks {
-		for i := range blk.preds {
-			pred := blk.preds[i].blk
-			if _, ok := b.blkVisited[pred]; ok || !pred.Valid() {
-				continue
-			} else if pred.reversePostOrder < blk.reversePostOrder {
-				// This means the edge is critical, and this pred is the trampoline and yet to be inserted.
-				// Split edge trampolines must come before the destination in reverse post-order.
-				b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, pred)
-				b.blkVisited[pred] = 0 // mark as inserted, the value is not used.
-			}
-		}
-
-		// Now that we've already added all the potential trampoline blocks incoming to this block,
-		// we can add this block itself.
-		b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, blk)
-		b.blkVisited[blk] = 0 // mark as inserted, the value is not used.
-
-		if len(blk.success) < 2 {
-			// There won't be critical edge originating from this block.
-			continue
-		} else if blk.currentInstr.opcode == OpcodeBrTable {
-			// We don't split critical edges here, because at the construction site of BrTable, we already split the edges.
-			continue
-		}
-
-		for sidx, succ := range blk.success {
-			if !succ.ReturnBlock() && // If the successor is a return block, we need to split the edge any way because we need "epilogue" to be inserted.
-				// Plus if there's no multiple incoming edges to this successor, (pred, succ) is not critical.
-				len(succ.preds) < 2 {
-				continue
-			}
-
-			// Otherwise, we are sure this is a critical edge. To modify the CFG, we need to find the predecessor info
-			// from the successor.
-			var predInfo *basicBlockPredecessorInfo
-			for i := range succ.preds { // This linear search should not be a problem since the number of predecessors should almost always small.
-				pred := &succ.preds[i]
-				if pred.blk == blk {
-					predInfo = pred
-					break
-				}
-			}
-
-			if predInfo == nil {
-				// This must be a bug in somewhere around branch manipulation.
-				panic("BUG: predecessor info not found while the successor exists in successors list")
-			}
-
-			if wazevoapi.SSALoggingEnabled {
-				fmt.Printf("trying to split edge from %d->%d at %s\n",
-					blk.ID(), succ.ID(), predInfo.branch.Format(b))
-			}
-
-			trampoline := b.splitCriticalEdge(blk, succ, predInfo)
-			// Update the successors slice because the target is no longer the original `succ`.
-			blk.success[sidx] = trampoline
-
-			if wazevoapi.SSAValidationEnabled {
-				trampolines = append(trampolines, trampoline)
-			}
-
-			if wazevoapi.SSALoggingEnabled {
-				fmt.Printf("edge split from %d->%d at %s as %d->%d->%d \n",
-					blk.ID(), succ.ID(), predInfo.branch.Format(b),
-					blk.ID(), trampoline.ID(), succ.ID())
-			}
-
-			fallthroughBranch := blk.currentInstr
-			if fallthroughBranch.opcode == OpcodeJump && fallthroughBranch.blk == trampoline {
-				// This can be lowered as fallthrough at the end of the block.
-				b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, trampoline)
-				b.blkVisited[trampoline] = 0 // mark as inserted, the value is not used.
-			} else {
-				uninsertedTrampolines = append(uninsertedTrampolines, trampoline)
-			}
-		}
-
-		for _, trampoline := range uninsertedTrampolines {
-			if trampoline.success[0].reversePostOrder <= trampoline.reversePostOrder { // "<=", not "<" because the target might be itself.
-				// This means the critical edge was backward, so we insert after the current block immediately.
-				b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, trampoline)
-				b.blkVisited[trampoline] = 0 // mark as inserted, the value is not used.
-			} // If the target is forward, we can wait to insert until the target is inserted.
-		}
-		uninsertedTrampolines = uninsertedTrampolines[:0] // Reuse the stack for the next block.
-	}
-
-	if wazevoapi.SSALoggingEnabled {
-		var bs []string
-		for _, blk := range b.reversePostOrderedBasicBlocks {
-			bs = append(bs, blk.Name())
-		}
-		fmt.Println("ordered blocks: ", strings.Join(bs, ", "))
-	}
-
-	if wazevoapi.SSAValidationEnabled {
-		for _, trampoline := range trampolines {
-			if _, ok := b.blkVisited[trampoline]; !ok {
-				panic("BUG: trampoline block not inserted: " + trampoline.FormatHeader(b))
-			}
-			trampoline.validate(b)
-		}
-	}
-
-	// Critical edges are split, so we fix the loop nesting forest.
-	buildLoopNestingForest(b)
-	buildDominatorTree(b)
-
-	// Reuse the stack for the next iteration.
-	b.blkStack2 = uninsertedTrampolines[:0]
-
-	// Now that we know the final placement of the blocks, we can explicitly mark the fallthrough jumps.
-	b.markFallthroughJumps()
-	b.doneBlockLayout = true
-}
-
-// markFallthroughJumps finds the fallthrough jumps and marks them as such.
-func (b *builder) markFallthroughJumps() {
-	l := len(b.reversePostOrderedBasicBlocks) - 1
-	for i, blk := range b.reversePostOrderedBasicBlocks {
-		if i < l {
-			cur := blk.currentInstr
-			if cur.opcode == OpcodeJump && cur.blk == b.reversePostOrderedBasicBlocks[i+1] {
-				cur.AsFallthroughJump()
-			}
-		}
-	}
-}
-
-// maybeInvertBranches inverts the branch instructions if it is likely possible to the fallthrough more likely with simple heuristics.
-// nextInRPO is the next block in the reverse post-order.
-//
-// Returns true if the branch is inverted for testing purpose.
-func maybeInvertBranches(now *basicBlock, nextInRPO *basicBlock) bool {
-	fallthroughBranch := now.currentInstr
-	if fallthroughBranch.opcode == OpcodeBrTable {
-		return false
-	}
-
-	condBranch := fallthroughBranch.prev
-	if condBranch == nil || (condBranch.opcode != OpcodeBrnz && condBranch.opcode != OpcodeBrz) {
-		return false
-	}
-
-	if len(fallthroughBranch.vs.View()) != 0 || len(condBranch.vs.View()) != 0 {
-		// If either one of them has arguments, we don't invert the branches.
-		return false
-	}
-
-	// So this block has two branches (a conditional branch followed by an unconditional branch) at the end.
-	// We can invert the condition of the branch if it makes the fallthrough more likely.
-
-	fallthroughTarget, condTarget := fallthroughBranch.blk.(*basicBlock), condBranch.blk.(*basicBlock)
-
-	if fallthroughTarget.loopHeader {
-		// First, if the tail's target is loopHeader, we don't need to do anything here,
-		// because the edge is likely to be critical edge for complex loops (e.g. loop with branches inside it).
-		// That means, we will split the edge in the end of LayoutBlocks function, and insert the trampoline block
-		// right after this block, which will be fallthrough in any way.
-		return false
-	} else if condTarget.loopHeader {
-		// On the other hand, if the condBranch's target is loopHeader, we invert the condition of the branch
-		// so that we could get the fallthrough to the trampoline block.
-		goto invert
-	}
-
-	if fallthroughTarget == nextInRPO {
-		// Also, if the tail's target is the next block in the reverse post-order, we don't need to do anything here,
-		// because if this is not critical edge, we would end up placing these two blocks adjacent to each other.
-		// Even if it is the critical edge, we place the trampoline block right after this block, which will be fallthrough in any way.
-		return false
-	} else if condTarget == nextInRPO {
-		// If the condBranch's target is the next block in the reverse post-order, we invert the condition of the branch
-		// so that we could get the fallthrough to the block.
-		goto invert
-	} else {
-		return false
-	}
-
-invert:
-	for i := range fallthroughTarget.preds {
-		pred := &fallthroughTarget.preds[i]
-		if pred.branch == fallthroughBranch {
-			pred.branch = condBranch
-			break
-		}
-	}
-	for i := range condTarget.preds {
-		pred := &condTarget.preds[i]
-		if pred.branch == condBranch {
-			pred.branch = fallthroughBranch
-			break
-		}
-	}
-
-	condBranch.InvertBrx()
-	condBranch.blk = fallthroughTarget
-	fallthroughBranch.blk = condTarget
-	if wazevoapi.SSALoggingEnabled {
-		fmt.Printf("inverting branches at %d->%d and %d->%d\n",
-			now.ID(), fallthroughTarget.ID(), now.ID(), condTarget.ID())
-	}
-
-	return true
-}
-
-// splitCriticalEdge splits the critical edge between the given predecessor (`pred`) and successor (owning `predInfo`).
-//
-// - `pred` is the source of the critical edge,
-// - `succ` is the destination of the critical edge,
-// - `predInfo` is the predecessor info in the succ.preds slice which represents the critical edge.
-//
-// Why splitting critical edges is important? See following links:
-//
-//   - https://en.wikipedia.org/wiki/Control-flow_graph
-//   - https://nickdesaulniers.github.io/blog/2023/01/27/critical-edge-splitting/
-//
-// The returned basic block is the trampoline block which is inserted to split the critical edge.
-func (b *builder) splitCriticalEdge(pred, succ *basicBlock, predInfo *basicBlockPredecessorInfo) *basicBlock {
-	// In the following, we convert the following CFG:
-	//
-	//     pred --(originalBranch)--> succ
-	//
-	// to the following CFG:
-	//
-	//     pred --(newBranch)--> trampoline --(originalBranch)-> succ
-	//
-	// where trampoline is a new basic block which is created to split the critical edge.
-
-	trampoline := b.allocateBasicBlock()
-	if int(trampoline.id) >= len(b.dominators) {
-		b.dominators = append(b.dominators, make([]*basicBlock, trampoline.id+1)...)
-	}
-	b.dominators[trampoline.id] = pred
-
-	originalBranch := predInfo.branch
-
-	// Replace originalBranch with the newBranch.
-	newBranch := b.AllocateInstruction()
-	newBranch.opcode = originalBranch.opcode
-	newBranch.blk = trampoline
-	switch originalBranch.opcode {
-	case OpcodeJump:
-	case OpcodeBrz, OpcodeBrnz:
-		originalBranch.opcode = OpcodeJump // Trampoline consists of one unconditional branch.
-		newBranch.v = originalBranch.v
-		originalBranch.v = ValueInvalid
-	default:
-		panic("BUG: critical edge shouldn't be originated from br_table")
-	}
-	swapInstruction(pred, originalBranch, newBranch)
-
-	// Replace the original branch with the new branch.
-	trampoline.rootInstr = originalBranch
-	trampoline.currentInstr = originalBranch
-	trampoline.success = append(trampoline.success, succ) // Do not use []*basicBlock{pred} because we might have already allocated the slice.
-	trampoline.preds = append(trampoline.preds,           // same as ^.
-		basicBlockPredecessorInfo{blk: pred, branch: newBranch})
-	b.Seal(trampoline)
-
-	// Update the original branch to point to the trampoline.
-	predInfo.blk = trampoline
-	predInfo.branch = originalBranch
-
-	if wazevoapi.SSAValidationEnabled {
-		trampoline.validate(b)
-	}
-
-	if len(trampoline.params) > 0 {
-		panic("trampoline should not have params")
-	}
-
-	// Assign the same order as the original block so that this will be placed before the actual destination.
-	trampoline.reversePostOrder = pred.reversePostOrder
-	return trampoline
-}
-
-// swapInstruction replaces `old` in the block `blk` with `New`.
-func swapInstruction(blk *basicBlock, old, New *Instruction) {
-	if blk.rootInstr == old {
-		blk.rootInstr = New
-		next := old.next
-		New.next = next
-		next.prev = New
-	} else {
-		if blk.currentInstr == old {
-			blk.currentInstr = New
-		}
-		prev := old.prev
-		prev.next, New.prev = New, prev
-		if next := old.next; next != nil {
-			New.next, next.prev = next, New
-		}
-	}
-	old.prev, old.next = nil, nil
-}
-
 // InsertUndefined implements Builder.InsertUndefined.
 func (b *builder) InsertUndefined() {
 	instr := b.AllocateInstruction()
diff --git a/internal/engine/wazevo/ssa/pass.go b/internal/engine/wazevo/ssa/pass.go
index 86ec5057..69afa7cb 100644
--- a/internal/engine/wazevo/ssa/pass.go
+++ b/internal/engine/wazevo/ssa/pass.go
@@ -13,6 +13,13 @@ import (
 // Note that passes suffixed with "Opt" are the optimization passes, meaning that they edit the instructions and blocks
 // while the other passes are not, like passEstimateBranchProbabilities does not edit them, but only calculates the additional information.
 func (b *builder) RunPasses() {
+	b.runPreBlockLayoutPasses()
+	b.runBlockLayoutPass()
+	b.runPostBlockLayoutPasses()
+	b.runFinalizingPasses()
+}
+
+func (b *builder) runPreBlockLayoutPasses() {
 	passSortSuccessors(b)
 	passDeadBlockEliminationOpt(b)
 	passRedundantPhiEliminationOpt(b)
@@ -34,7 +41,38 @@ func (b *builder) RunPasses() {
 
 	// passDeadCodeEliminationOpt could be more accurate if we do this after other optimizations.
 	passDeadCodeEliminationOpt(b)
-	b.donePasses = true
+	b.donePreBlockLayoutPasses = true
+}
+
+func (b *builder) runBlockLayoutPass() {
+	if !b.donePreBlockLayoutPasses {
+		panic("runBlockLayoutPass must be called after all pre passes are done")
+	}
+	passLayoutBlocks(b)
+	b.doneBlockLayout = true
+}
+
+// runPostBlockLayoutPasses runs the post block layout passes. After this point, CFG is somewhat stable,
+// but still can be modified before finalizing passes. At this point, critical edges are split by passLayoutBlocks.
+func (b *builder) runPostBlockLayoutPasses() {
+	if !b.doneBlockLayout {
+		panic("runPostBlockLayoutPasses must be called after block layout pass is done")
+	}
+	// TODO: Do more. e.g. tail duplication, loop unrolling, etc.
+
+	b.donePostBlockLayoutPasses = true
+}
+
+// runFinalizingPasses runs the finalizing passes. After this point, CFG should not be modified.
+func (b *builder) runFinalizingPasses() {
+	if !b.donePostBlockLayoutPasses {
+		panic("runFinalizingPasses must be called after post block layout passes are done")
+	}
+	// Critical edges are split, so we fix the loop nesting forest.
+	passBuildLoopNestingForest(b)
+	passBuildDominatorTree(b)
+	// Now that we know the final placement of the blocks, we can explicitly mark the fallthrough jumps.
+	b.markFallthroughJumps()
 }
 
 // passDeadBlockEliminationOpt searches the unreachable blocks, and sets the basicBlock.invalid flag true if so.
diff --git a/internal/engine/wazevo/ssa/pass_blk_layouts.go b/internal/engine/wazevo/ssa/pass_blk_layouts.go
new file mode 100644
index 00000000..9068180a
--- /dev/null
+++ b/internal/engine/wazevo/ssa/pass_blk_layouts.go
@@ -0,0 +1,335 @@
+package ssa
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/tetratelabs/wazero/internal/engine/wazevo/wazevoapi"
+)
+
+// passLayoutBlocks implements Builder.LayoutBlocks. This re-organizes builder.reversePostOrderedBasicBlocks.
+//
+// TODO: there are tons of room for improvement here. e.g. LLVM has BlockPlacementPass using BlockFrequencyInfo,
+// BranchProbabilityInfo, and LoopInfo to do a much better job. Also, if we have the profiling instrumentation
+// like ball-larus algorithm, then we could do profile-guided optimization. Basically all of them are trying
+// to maximize the fall-through opportunities which is most efficient.
+//
+// Here, fallthrough happens when a block ends with jump instruction whose target is the right next block in the
+// builder.reversePostOrderedBasicBlocks.
+//
+// Currently, we just place blocks using the DFS reverse post-order of the dominator tree with the heuristics:
+//  1. a split edge trampoline towards a loop header will be placed as a fallthrough.
+//  2. we invert the brz and brnz if it makes the fallthrough more likely.
+//
+// This heuristic is done in maybeInvertBranches function.
+func passLayoutBlocks(b *builder) {
+	b.clearBlkVisited()
+
+	// We might end up splitting critical edges which adds more basic blocks,
+	// so we store the currently existing basic blocks in nonSplitBlocks temporarily.
+	// That way we can iterate over the original basic blocks while appending new ones into reversePostOrderedBasicBlocks.
+	nonSplitBlocks := b.blkStack[:0]
+	for i, blk := range b.reversePostOrderedBasicBlocks {
+		if !blk.Valid() {
+			continue
+		}
+		nonSplitBlocks = append(nonSplitBlocks, blk)
+		if i != len(b.reversePostOrderedBasicBlocks)-1 {
+			_ = maybeInvertBranches(blk, b.reversePostOrderedBasicBlocks[i+1])
+		}
+	}
+
+	var trampolines []*basicBlock
+
+	// Reset the order slice since we update on the fly by splitting critical edges.
+	b.reversePostOrderedBasicBlocks = b.reversePostOrderedBasicBlocks[:0]
+	uninsertedTrampolines := b.blkStack2[:0]
+	for _, blk := range nonSplitBlocks {
+		for i := range blk.preds {
+			pred := blk.preds[i].blk
+			if _, ok := b.blkVisited[pred]; ok || !pred.Valid() {
+				continue
+			} else if pred.reversePostOrder < blk.reversePostOrder {
+				// This means the edge is critical, and this pred is the trampoline and yet to be inserted.
+				// Split edge trampolines must come before the destination in reverse post-order.
+				b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, pred)
+				b.blkVisited[pred] = 0 // mark as inserted, the value is not used.
+			}
+		}
+
+		// Now that we've already added all the potential trampoline blocks incoming to this block,
+		// we can add this block itself.
+		b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, blk)
+		b.blkVisited[blk] = 0 // mark as inserted, the value is not used.
+
+		if len(blk.success) < 2 {
+			// There won't be critical edge originating from this block.
+			continue
+		} else if blk.currentInstr.opcode == OpcodeBrTable {
+			// We don't split critical edges here, because at the construction site of BrTable, we already split the edges.
+			continue
+		}
+
+		for sidx, succ := range blk.success {
+			if !succ.ReturnBlock() && // If the successor is a return block, we need to split the edge any way because we need "epilogue" to be inserted.
+				// Plus if there's no multiple incoming edges to this successor, (pred, succ) is not critical.
+				len(succ.preds) < 2 {
+				continue
+			}
+
+			// Otherwise, we are sure this is a critical edge. To modify the CFG, we need to find the predecessor info
+			// from the successor.
+			var predInfo *basicBlockPredecessorInfo
+			for i := range succ.preds { // This linear search should not be a problem since the number of predecessors should almost always small.
+				pred := &succ.preds[i]
+				if pred.blk == blk {
+					predInfo = pred
+					break
+				}
+			}
+
+			if predInfo == nil {
+				// This must be a bug in somewhere around branch manipulation.
+				panic("BUG: predecessor info not found while the successor exists in successors list")
+			}
+
+			if wazevoapi.SSALoggingEnabled {
+				fmt.Printf("trying to split edge from %d->%d at %s\n",
+					blk.ID(), succ.ID(), predInfo.branch.Format(b))
+			}
+
+			trampoline := b.splitCriticalEdge(blk, succ, predInfo)
+			// Update the successors slice because the target is no longer the original `succ`.
+			blk.success[sidx] = trampoline
+
+			if wazevoapi.SSAValidationEnabled {
+				trampolines = append(trampolines, trampoline)
+			}
+
+			if wazevoapi.SSALoggingEnabled {
+				fmt.Printf("edge split from %d->%d at %s as %d->%d->%d \n",
+					blk.ID(), succ.ID(), predInfo.branch.Format(b),
+					blk.ID(), trampoline.ID(), succ.ID())
+			}
+
+			fallthroughBranch := blk.currentInstr
+			if fallthroughBranch.opcode == OpcodeJump && fallthroughBranch.blk == trampoline {
+				// This can be lowered as fallthrough at the end of the block.
+				b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, trampoline)
+				b.blkVisited[trampoline] = 0 // mark as inserted, the value is not used.
+			} else {
+				uninsertedTrampolines = append(uninsertedTrampolines, trampoline)
+			}
+		}
+
+		for _, trampoline := range uninsertedTrampolines {
+			if trampoline.success[0].reversePostOrder <= trampoline.reversePostOrder { // "<=", not "<" because the target might be itself.
+				// This means the critical edge was backward, so we insert after the current block immediately.
+				b.reversePostOrderedBasicBlocks = append(b.reversePostOrderedBasicBlocks, trampoline)
+				b.blkVisited[trampoline] = 0 // mark as inserted, the value is not used.
+			} // If the target is forward, we can wait to insert until the target is inserted.
+		}
+		uninsertedTrampolines = uninsertedTrampolines[:0] // Reuse the stack for the next block.
+	}
+
+	if wazevoapi.SSALoggingEnabled {
+		var bs []string
+		for _, blk := range b.reversePostOrderedBasicBlocks {
+			bs = append(bs, blk.Name())
+		}
+		fmt.Println("ordered blocks: ", strings.Join(bs, ", "))
+	}
+
+	if wazevoapi.SSAValidationEnabled {
+		for _, trampoline := range trampolines {
+			if _, ok := b.blkVisited[trampoline]; !ok {
+				panic("BUG: trampoline block not inserted: " + trampoline.FormatHeader(b))
+			}
+			trampoline.validate(b)
+		}
+	}
+
+	// Reuse the stack for the next iteration.
+	b.blkStack2 = uninsertedTrampolines[:0]
+}
+
+// markFallthroughJumps finds the fallthrough jumps and marks them as such.
+func (b *builder) markFallthroughJumps() {
+	l := len(b.reversePostOrderedBasicBlocks) - 1
+	for i, blk := range b.reversePostOrderedBasicBlocks {
+		if i < l {
+			cur := blk.currentInstr
+			if cur.opcode == OpcodeJump && cur.blk == b.reversePostOrderedBasicBlocks[i+1] {
+				cur.AsFallthroughJump()
+			}
+		}
+	}
+}
+
+// maybeInvertBranches inverts the branch instructions if it is likely possible to the fallthrough more likely with simple heuristics.
+// nextInRPO is the next block in the reverse post-order.
+//
+// Returns true if the branch is inverted for testing purpose.
+func maybeInvertBranches(now *basicBlock, nextInRPO *basicBlock) bool {
+	fallthroughBranch := now.currentInstr
+	if fallthroughBranch.opcode == OpcodeBrTable {
+		return false
+	}
+
+	condBranch := fallthroughBranch.prev
+	if condBranch == nil || (condBranch.opcode != OpcodeBrnz && condBranch.opcode != OpcodeBrz) {
+		return false
+	}
+
+	if len(fallthroughBranch.vs.View()) != 0 || len(condBranch.vs.View()) != 0 {
+		// If either one of them has arguments, we don't invert the branches.
+		return false
+	}
+
+	// So this block has two branches (a conditional branch followed by an unconditional branch) at the end.
+	// We can invert the condition of the branch if it makes the fallthrough more likely.
+
+	fallthroughTarget, condTarget := fallthroughBranch.blk.(*basicBlock), condBranch.blk.(*basicBlock)
+
+	if fallthroughTarget.loopHeader {
+		// First, if the tail's target is loopHeader, we don't need to do anything here,
+		// because the edge is likely to be critical edge for complex loops (e.g. loop with branches inside it).
+		// That means, we will split the edge in the end of LayoutBlocks function, and insert the trampoline block
+		// right after this block, which will be fallthrough in any way.
+		return false
+	} else if condTarget.loopHeader {
+		// On the other hand, if the condBranch's target is loopHeader, we invert the condition of the branch
+		// so that we could get the fallthrough to the trampoline block.
+		goto invert
+	}
+
+	if fallthroughTarget == nextInRPO {
+		// Also, if the tail's target is the next block in the reverse post-order, we don't need to do anything here,
+		// because if this is not critical edge, we would end up placing these two blocks adjacent to each other.
+		// Even if it is the critical edge, we place the trampoline block right after this block, which will be fallthrough in any way.
+		return false
+	} else if condTarget == nextInRPO {
+		// If the condBranch's target is the next block in the reverse post-order, we invert the condition of the branch
+		// so that we could get the fallthrough to the block.
+		goto invert
+	} else {
+		return false
+	}
+
+invert:
+	for i := range fallthroughTarget.preds {
+		pred := &fallthroughTarget.preds[i]
+		if pred.branch == fallthroughBranch {
+			pred.branch = condBranch
+			break
+		}
+	}
+	for i := range condTarget.preds {
+		pred := &condTarget.preds[i]
+		if pred.branch == condBranch {
+			pred.branch = fallthroughBranch
+			break
+		}
+	}
+
+	condBranch.InvertBrx()
+	condBranch.blk = fallthroughTarget
+	fallthroughBranch.blk = condTarget
+	if wazevoapi.SSALoggingEnabled {
+		fmt.Printf("inverting branches at %d->%d and %d->%d\n",
+			now.ID(), fallthroughTarget.ID(), now.ID(), condTarget.ID())
+	}
+
+	return true
+}
+
+// splitCriticalEdge splits the critical edge between the given predecessor (`pred`) and successor (owning `predInfo`).
+//
+// - `pred` is the source of the critical edge,
+// - `succ` is the destination of the critical edge,
+// - `predInfo` is the predecessor info in the succ.preds slice which represents the critical edge.
+//
+// Why splitting critical edges is important? See following links:
+//
+//   - https://en.wikipedia.org/wiki/Control-flow_graph
+//   - https://nickdesaulniers.github.io/blog/2023/01/27/critical-edge-splitting/
+//
+// The returned basic block is the trampoline block which is inserted to split the critical edge.
+func (b *builder) splitCriticalEdge(pred, succ *basicBlock, predInfo *basicBlockPredecessorInfo) *basicBlock {
+	// In the following, we convert the following CFG:
+	//
+	//     pred --(originalBranch)--> succ
+	//
+	// to the following CFG:
+	//
+	//     pred --(newBranch)--> trampoline --(originalBranch)-> succ
+	//
+	// where trampoline is a new basic block which is created to split the critical edge.
+
+	trampoline := b.allocateBasicBlock()
+	if int(trampoline.id) >= len(b.dominators) {
+		b.dominators = append(b.dominators, make([]*basicBlock, trampoline.id+1)...)
+	}
+	b.dominators[trampoline.id] = pred
+
+	originalBranch := predInfo.branch
+
+	// Replace originalBranch with the newBranch.
+	newBranch := b.AllocateInstruction()
+	newBranch.opcode = originalBranch.opcode
+	newBranch.blk = trampoline
+	switch originalBranch.opcode {
+	case OpcodeJump:
+	case OpcodeBrz, OpcodeBrnz:
+		originalBranch.opcode = OpcodeJump // Trampoline consists of one unconditional branch.
+		newBranch.v = originalBranch.v
+		originalBranch.v = ValueInvalid
+	default:
+		panic("BUG: critical edge shouldn't be originated from br_table")
+	}
+	swapInstruction(pred, originalBranch, newBranch)
+
+	// Replace the original branch with the new branch.
+	trampoline.rootInstr = originalBranch
+	trampoline.currentInstr = originalBranch
+	trampoline.success = append(trampoline.success, succ) // Do not use []*basicBlock{pred} because we might have already allocated the slice.
+	trampoline.preds = append(trampoline.preds,           // same as ^.
+		basicBlockPredecessorInfo{blk: pred, branch: newBranch})
+	b.Seal(trampoline)
+
+	// Update the original branch to point to the trampoline.
+	predInfo.blk = trampoline
+	predInfo.branch = originalBranch
+
+	if wazevoapi.SSAValidationEnabled {
+		trampoline.validate(b)
+	}
+
+	if len(trampoline.params) > 0 {
+		panic("trampoline should not have params")
+	}
+
+	// Assign the same order as the original block so that this will be placed before the actual destination.
+	trampoline.reversePostOrder = pred.reversePostOrder
+	return trampoline
+}
+
+// swapInstruction replaces `old` in the block `blk` with `New`.
+func swapInstruction(blk *basicBlock, old, New *Instruction) {
+	if blk.rootInstr == old {
+		blk.rootInstr = New
+		next := old.next
+		New.next = next
+		next.prev = New
+	} else {
+		if blk.currentInstr == old {
+			blk.currentInstr = New
+		}
+		prev := old.prev
+		prev.next, New.prev = New, prev
+		if next := old.next; next != nil {
+			New.next, next.prev = next, New
+		}
+	}
+	old.prev, old.next = nil, nil
+}
diff --git a/internal/engine/wazevo/ssa/pass_cfg.go b/internal/engine/wazevo/ssa/pass_cfg.go
index c39a75b5..50cb9c47 100644
--- a/internal/engine/wazevo/ssa/pass_cfg.go
+++ b/internal/engine/wazevo/ssa/pass_cfg.go
@@ -167,7 +167,7 @@ func subPassLoopDetection(b *builder) {
 
 // buildLoopNestingForest builds the loop nesting forest for the function.
 // This must be called after branch splitting since it relies on the CFG.
-func buildLoopNestingForest(b *builder) {
+func passBuildLoopNestingForest(b *builder) {
 	ent := b.entryBlk()
 	doms := b.dominators
 	for _, blk := range b.reversePostOrderedBasicBlocks {
@@ -208,8 +208,8 @@ type dominatorSparseTree struct {
 	table        [][]int
 }
 
-// buildDominatorTree builds the dominator tree for the function, and constructs builder.sparseTree.
-func buildDominatorTree(b *builder) {
+// passBuildDominatorTree builds the dominator tree for the function, and constructs builder.sparseTree.
+func passBuildDominatorTree(b *builder) {
 	// First we materialize the children of each node in the dominator tree.
 	idoms := b.dominators
 	for _, blk := range b.reversePostOrderedBasicBlocks {
diff --git a/internal/engine/wazevo/ssa/vs.go b/internal/engine/wazevo/ssa/vs.go
index bcbf7a30..bcf83cbf 100644
--- a/internal/engine/wazevo/ssa/vs.go
+++ b/internal/engine/wazevo/ssa/vs.go
@@ -52,7 +52,7 @@ func (v Value) formatWithType(b Builder) (ret string) {
 	}
 
 	if wazevoapi.SSALoggingEnabled { // This is useful to check live value analysis bugs.
-		if bd := b.(*builder); bd.donePasses {
+		if bd := b.(*builder); bd.donePostBlockLayoutPasses {
 			id := v.ID()
 			ret += fmt.Sprintf("(ref=%d)", bd.valueRefCounts[id])
 		}


Test patch:
diff --git a/internal/engine/wazevo/backend/backend_test.go b/internal/engine/wazevo/backend/backend_test.go
index fe775615..0cf8384f 100644
--- a/internal/engine/wazevo/backend/backend_test.go
+++ b/internal/engine/wazevo/backend/backend_test.go
@@ -2449,12 +2449,6 @@ L2 (SSA Block: blk2):
 				fmt.Println(ssab.Format())
 			}
 
-			ssab.LayoutBlocks()
-			if verbose {
-				fmt.Println("============ SSA after block layout ============")
-				fmt.Println(ssab.Format())
-			}
-
 			// Lowers the SSA to ISA specific code.
 			be.Lower()
 			if verbose {
diff --git a/internal/engine/wazevo/frontend/frontend_test.go b/internal/engine/wazevo/frontend/frontend_test.go
index 8e12e30f..511465bd 100644
--- a/internal/engine/wazevo/frontend/frontend_test.go
+++ b/internal/engine/wazevo/frontend/frontend_test.go
@@ -26,9 +26,9 @@ func TestCompiler_LowerToSSA(t *testing.T) {
 		targetIndex wasm.Index
 		// exp is the *unoptimized* expected SSA IR for the function m.FunctionSection[targetIndex].
 		exp string
-		// expAfterOpt is not empty when we want to check the result after optimization passes.
-		expAfterOpt string
-		features    api.CoreFeatures
+		// expAfterPasses is not empty when we want to check the result after SSA passes.
+		expAfterPasses string
+		features       api.CoreFeatures
 	}{
 		{
 			name: "empty", m: testcases.Empty.Module,
@@ -101,7 +101,7 @@ blk0: (exec_ctx:i64, module_ctx:i64)
 	v5:f64 = F64const 0.000000
 	Jump blk_ret
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64)
 	Jump blk_ret
 `,
@@ -127,7 +127,7 @@ blk0: (exec_ctx:i64, module_ctx:i64, v2:i32)
 	v3:i32 = Iconst_32 0x0
 	Jump blk_ret, v2, v2
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32)
 	Jump blk_ret, v2, v2
 `,
@@ -156,7 +156,7 @@ blk0: (exec_ctx:i64, module_ctx:i64, v2:i64, v3:f32, v4:f64)
 	v22:f64 = Fmin v21, v4
 	Jump blk_ret, v10, v16, v22
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i64, v3:f32, v4:f64)
 	v9:i64 = Iadd v2, v2
 	v10:i64 = Isub v9, v2
@@ -240,9 +240,9 @@ blk1: () <-- (blk0,blk1)
 
 blk2: ()
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64)
-	Jump blk1
+	Jump fallthrough
 
 blk1: () <-- (blk0,blk1)
 	Jump blk1
@@ -265,12 +265,12 @@ blk1: () <-- (blk0,blk1)
 
 blk2: ()
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 signatures:
 	sig2: i64_v
 
 blk0: (exec_ctx:i64, module_ctx:i64)
-	Jump blk1
+	Jump fallthrough
 
 blk1: () <-- (blk0,blk1)
 	v2:i64 = Load exec_ctx, 0x58
@@ -294,14 +294,17 @@ blk2: ()
 blk3: () <-- (blk1)
 	Return
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64)
-	Jump blk1
+	Jump fallthrough
 
-blk1: () <-- (blk0,blk1)
+blk1: () <-- (blk0,blk4)
 	v2:i32 = Iconst_32 0x1
-	Brnz v2, blk1
-	Jump blk3
+	Brz v2, blk3
+	Jump fallthrough
+
+blk4: () <-- (blk1)
+	Jump blk1
 
 blk3: () <-- (blk1)
 	Return
@@ -322,9 +325,9 @@ blk1: () <-- (blk0)
 
 blk2: ()
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64)
-	Jump blk1
+	Jump fallthrough
 
 blk1: () <-- (blk0)
 	Jump blk_ret
@@ -385,18 +388,18 @@ blk2: () <-- (blk0)
 blk3: () <-- (blk2)
 	Jump blk_ret, v2
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64)
 	v2:i32 = Iconst_32 0x0
 	v4:i32 = Iconst_32 0x0
 	Brz v2, blk2
-	Jump blk1
+	Jump fallthrough
 
 blk1: () <-- (blk0)
 	Return v4
 
 blk2: () <-- (blk0)
-	Jump blk3
+	Jump fallthrough
 
 blk3: () <-- (blk2)
 	Jump blk_ret, v2
@@ -420,16 +423,16 @@ blk2: () <-- (blk0)
 blk3: (v5:i32) <-- (blk1,blk2)
 	Jump blk_ret, v5
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i32)
 	Brz v2, blk2
-	Jump blk1
+	Jump fallthrough
 
 blk1: () <-- (blk0)
 	Jump blk3, v2
 
 blk2: () <-- (blk0)
-	Jump blk3, v3
+	Jump fallthrough, v3
 
 blk3: (v5:i32) <-- (blk1,blk2)
 	Jump blk_ret, v5
@@ -448,9 +451,9 @@ blk1: (v4:i32) <-- (blk0)
 
 blk2: ()
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32)
-	Jump blk1
+	Jump fallthrough
 
 blk1: () <-- (blk0)
 	Return v2
@@ -477,23 +480,26 @@ blk3: () <-- (blk4)
 blk4: () <-- (blk1)
 	Jump blk3
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32)
+	Jump fallthrough
+
+blk1: () <-- (blk0,blk5)
+	Brz v2, blk4
+	Jump fallthrough
+
+blk5: () <-- (blk1)
 	Jump blk1
 
-blk1: () <-- (blk0,blk1)
-	Brnz v2, blk1
-	Jump blk4
+blk4: () <-- (blk1)
+	Jump fallthrough
+
+blk3: () <-- (blk4)
+	Jump fallthrough
 
 blk2: () <-- (blk3)
 	v4:i32 = Iconst_32 0x0
 	Jump blk_ret, v4
-
-blk3: () <-- (blk4)
-	Jump blk2
-
-blk4: () <-- (blk1)
-	Jump blk3
 `,
 		},
 		{
@@ -516,20 +522,23 @@ blk3: () <-- (blk4)
 blk4: () <-- (blk1)
 	Jump blk3
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32)
-	Jump blk1, v2
+	Jump fallthrough, v2
 
 blk1: (v3:i32) <-- (blk0,blk3)
-	Brnz v3, blk_ret
+	Brnz v3, blk5
 	Jump blk4
 
+blk5: () <-- (blk1)
+	Jump blk_ret
+
+blk4: () <-- (blk1)
+	Jump fallthrough
+
 blk3: () <-- (blk4)
 	v4:i32 = Iconst_32 0x1
 	Jump blk1, v4
-
-blk4: () <-- (blk1)
-	Jump blk3
 `,
 		},
 		{
@@ -604,7 +613,7 @@ blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i32, v4:i64, v5:i64)
 	v25:i32 = Icmp ge_u, v4, v5
 	Jump blk_ret, v6, v7, v8, v9, v10, v11, v12, v13, v14, v15, v16, v17, v18, v19, v20, v21, v22, v23, v24, v25
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i32, v4:i64, v5:i64)
 	v6:i32 = Icmp eq, v2, v3
 	v7:i32 = Icmp eq, v4, v5
@@ -768,20 +777,23 @@ blk2: (v6:i32) <-- (blk3)
 blk3: () <-- (blk1)
 	Jump blk2, v4
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i32)
-	Jump blk1
+	Jump fallthrough
 
-blk1: () <-- (blk0,blk1)
+blk1: () <-- (blk0,blk4)
 	v7:i32 = Iconst_32 0x1
-	Brnz v7, blk1
-	Jump blk3
+	Brz v7, blk3
+	Jump fallthrough
 
-blk2: () <-- (blk3)
-	Jump blk_ret, v2
+blk4: () <-- (blk1)
+	Jump blk1
 
 blk3: () <-- (blk1)
-	Jump blk2
+	Jump fallthrough
+
+blk2: () <-- (blk3)
+	Jump blk_ret, v2
 `,
 		},
 		{
@@ -792,7 +804,7 @@ blk3: () <-- (blk1)
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i64, v4:f32, v5:f64, v6:i32, v7:i64, v8:f32, v9:f64, v10:i32, v11:i64, v12:f32, v13:f64, v14:i32, v15:i64, v16:f32, v17:f64, v18:i32, v19:i64, v20:f32, v21:f64, v22:i32, v23:i64, v24:f32, v25:f64, v26:i32, v27:i64, v28:f32, v29:f64, v30:i32, v31:i64, v32:f32, v33:f64, v34:i32, v35:i64, v36:f32, v37:f64, v38:i32, v39:i64, v40:f32, v41:f64)
 	Jump blk_ret, v2, v11, v20, v29
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i64, v4:f32, v5:f64, v6:i32, v7:i64, v8:f32, v9:f64, v10:i32, v11:i64, v12:f32, v13:f64, v14:i32, v15:i64, v16:f32, v17:f64, v18:i32, v19:i64, v20:f32, v21:f64, v22:i32, v23:i64, v24:f32, v25:f64, v26:i32, v27:i64, v28:f32, v29:f64, v30:i32, v31:i64, v32:f32, v33:f64, v34:i32, v35:i64, v36:f32, v37:f64, v38:i32, v39:i64, v40:f32, v41:f64)
 	Jump blk_ret, v2, v11, v20, v29
 `,
@@ -805,7 +817,7 @@ blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i64, v4:f32, v5:f64, v6:i32, v7:
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i64, v4:f32, v5:f64)
 	Jump blk_ret, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i64, v4:f32, v5:f64)
 	Jump blk_ret, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5, v2, v3, v4, v5
 `,
@@ -818,7 +830,7 @@ blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i64, v4:f32, v5:f64)
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i64, v4:f32, v5:f64, v6:i32, v7:i64, v8:f32, v9:f64, v10:i32, v11:i64, v12:f32, v13:f64, v14:i32, v15:i64, v16:f32, v17:f64, v18:i32, v19:i64, v20:f32, v21:f64, v22:i32, v23:i64, v24:f32, v25:f64, v26:i32, v27:i64, v28:f32, v29:f64, v30:i32, v31:i64, v32:f32, v33:f64, v34:i32, v35:i64, v36:f32, v37:f64, v38:i32, v39:i64, v40:f32, v41:f64)
 	Jump blk_ret, v41, v40, v39, v38, v37, v36, v35, v34, v33, v32, v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, v15, v14, v13, v12, v11, v10, v9, v8, v7, v6, v5, v4, v3, v2
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i64, v4:f32, v5:f64, v6:i32, v7:i64, v8:f32, v9:f64, v10:i32, v11:i64, v12:f32, v13:f64, v14:i32, v15:i64, v16:f32, v17:f64, v18:i32, v19:i64, v20:f32, v21:f64, v22:i32, v23:i64, v24:f32, v25:f64, v26:i32, v27:i64, v28:f32, v29:f64, v30:i32, v31:i64, v32:f32, v33:f64, v34:i32, v35:i64, v36:f32, v37:f64, v38:i32, v39:i64, v40:f32, v41:f64)
 	Jump blk_ret, v41, v40, v39, v38, v37, v36, v35, v34, v33, v32, v31, v30, v29, v28, v27, v26, v25, v24, v23, v22, v21, v20, v19, v18, v17, v16, v15, v14, v13, v12, v11, v10, v9, v8, v7, v6, v5, v4, v3, v2
 `,
@@ -979,7 +991,7 @@ blk3: () <-- (blk2)
 	v11:i32 = Iadd v7, v10
 	Jump blk_ret, v11
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 signatures:
 	sig0: i64i64i32_i32
 
@@ -987,13 +999,13 @@ blk0: (exec_ctx:i64, module_ctx:i64, v2:i32)
 	v3:i32 = Iconst_32 0x2
 	v4:i32 = Icmp lt_s, v2, v3
 	Brz v4, blk2
-	Jump blk1
+	Jump fallthrough
 
 blk1: () <-- (blk0)
 	Return v2
 
 blk2: () <-- (blk0)
-	Jump blk3
+	Jump fallthrough
 
 blk3: () <-- (blk2)
 	v5:i32 = Iconst_32 0x1
@@ -1075,7 +1087,7 @@ blk3: (v7:i32) <-- (blk1,blk2)
 	v15:i32 = Load v14, 0x0
 	Jump blk_ret, v15
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 signatures:
 	sig1: i64i64_v
 
@@ -1083,7 +1095,7 @@ blk0: (exec_ctx:i64, module_ctx:i64, v2:i32)
 	v3:i32 = Iconst_32 0x0
 	v4:i32 = Icmp eq, v2, v3
 	Brz v4, blk2
-	Jump blk1
+	Jump fallthrough
 
 blk1: () <-- (blk0)
 	Store module_ctx, exec_ctx, 0x8
@@ -1091,7 +1103,7 @@ blk1: () <-- (blk0)
 	Jump blk3
 
 blk2: () <-- (blk0)
-	Jump blk3
+	Jump fallthrough
 
 blk3: () <-- (blk1,blk2)
 	v8:i64 = Iconst_64 0x4
@@ -1229,7 +1241,7 @@ blk0: (exec_ctx:i64, module_ctx:i64)
 	v9:f64 = Load module_ctx, 0x40
 	Jump blk_ret, v2, v3, v4, v5, v6, v7, v8, v9
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 signatures:
 	sig1: i64i64_v
 
@@ -1290,7 +1302,7 @@ blk0: (exec_ctx:i64, module_ctx:i64)
 	v30:i32 = Ushr v28, v29
 	Jump blk_ret, v4, v12, v22, v30
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 signatures:
 	sig0: i64i64_i32
 	sig2: i64i32_i32
@@ -1344,7 +1356,7 @@ blk0: (exec_ctx:i64, module_ctx:i64)
 	v14:i64 = Uload32 module_ctx, 0x10
 	Jump blk_ret, v4, v9, v12
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 signatures:
 	sig1: i64i32_i32
 
@@ -1504,61 +1516,60 @@ blk13: () <-- (blk0)
 	Jump blk6, v3
 `,
 
-			// TODO: these trivial two hop jumps should be optimized away.
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:i32, v3:i32)
 	BrTable v2, [blk7, blk8, blk9, blk10, blk11, blk12, blk13]
 
-blk1: () <-- (blk12)
-	v20:i32 = Iconst_32 0x10
-	v21:i32 = Iadd v3, v20
-	Return v21
-
-blk2: () <-- (blk11)
-	v18:i32 = Iconst_32 0xf
-	v19:i32 = Iadd v3, v18
-	Return v19
-
-blk3: () <-- (blk10)
-	v16:i32 = Iconst_32 0xe
-	v17:i32 = Iadd v3, v16
-	Return v17
+blk7: () <-- (blk0)
+	Jump blk6
 
-blk4: () <-- (blk9)
-	v14:i32 = Iconst_32 0xd
-	v15:i32 = Iadd v3, v14
-	Return v15
+blk8: () <-- (blk0)
+	Jump fallthrough
 
 blk5: () <-- (blk8)
 	v12:i32 = Iconst_32 0xc
 	v13:i32 = Iadd v3, v12
 	Return v13
 
-blk6: () <-- (blk7,blk13)
-	v10:i32 = Iconst_32 0xb
-	v11:i32 = Iadd v3, v10
-	Return v11
-
-blk7: () <-- (blk0)
-	Jump blk6
-
-blk8: () <-- (blk0)
-	Jump blk5
-
 blk9: () <-- (blk0)
-	Jump blk4
+	Jump fallthrough
+
+blk4: () <-- (blk9)
+	v14:i32 = Iconst_32 0xd
+	v15:i32 = Iadd v3, v14
+	Return v15
 
 blk10: () <-- (blk0)
-	Jump blk3
+	Jump fallthrough
+
+blk3: () <-- (blk10)
+	v16:i32 = Iconst_32 0xe
+	v17:i32 = Iadd v3, v16
+	Return v17
 
 blk11: () <-- (blk0)
-	Jump blk2
+	Jump fallthrough
+
+blk2: () <-- (blk11)
+	v18:i32 = Iconst_32 0xf
+	v19:i32 = Iadd v3, v18
+	Return v19
 
 blk12: () <-- (blk0)
-	Jump blk1
+	Jump fallthrough
+
+blk1: () <-- (blk12)
+	v20:i32 = Iconst_32 0x10
+	v21:i32 = Iadd v3, v20
+	Return v21
 
 blk13: () <-- (blk0)
-	Jump blk6
+	Jump fallthrough
+
+blk6: () <-- (blk7,blk13)
+	v10:i32 = Iconst_32 0xb
+	v11:i32 = Iadd v3, v10
+	Return v11
 `,
 		},
 		{
@@ -1587,25 +1598,25 @@ blk4: () <-- (blk2,blk3)
 	v12:i64 = Iconst_64 0x0
 	Jump blk1, v12
 `,
-			expAfterOpt: `
+			expAfterPasses: `
 blk0: (exec_ctx:i64, module_ctx:i64, v2:f64, v3:f64, v4:f64)
 	v6:i32 = Load module_ctx, 0x10
 	v7:i32 = Iconst_32 0x10
 	v8:i32 = Ushr v6, v7
 	Brz v8, blk3
-	Jump blk2
-
-blk1: () <-- (blk4)
-	Jump blk_ret
+	Jump fallthrough
 
 blk2: () <-- (blk0)
 	Jump blk4
 
 blk3: () <-- (blk0)
-	Jump blk4
+	Jump fallthrough
 
 blk4: () <-- (blk2,blk3)
-	Jump blk1
+	Jump fallthrough
+
+blk1: () <-- (blk4)
+	Jump blk_ret
 `,
 		},
 		{
@@ -2840,13 +2851,10 @@ blk0: (exec_ctx:i64, module_ctx:i64)
 			require.Equal(t, tc.exp, actual)
 
 			b.RunPasses()
-			if expAfterOpt := tc.expAfterOpt; expAfterOpt != "" {
+			if expAfterOpt := tc.expAfterPasses; expAfterOpt != "" {
 				actualAfterOpt := fc.formatBuilder()
 				require.Equal(t, expAfterOpt, actualAfterOpt)
 			}
-
-			// Dry-run without checking the results of LayoutBlocks function.
-			b.LayoutBlocks()
 		})
 	}
 }
diff --git a/internal/engine/wazevo/ssa/builder_test.go b/internal/engine/wazevo/ssa/pass_blk_layouts_test.go
similarity index 99%
rename from internal/engine/wazevo/ssa/builder_test.go
rename to internal/engine/wazevo/ssa/pass_blk_layouts_test.go
index 26cbdf6f..26a5d529 100644
--- a/internal/engine/wazevo/ssa/builder_test.go
+++ b/internal/engine/wazevo/ssa/pass_blk_layouts_test.go
@@ -612,9 +612,8 @@ func TestBuilder_LayoutBlocks(t *testing.T) {
 			b := NewBuilder().(*builder)
 			tc.setup(b)
 
-			b.RunPasses() // LayoutBlocks() must be called after RunPasses().
-			b.LayoutBlocks()
-
+			b.runPreBlockLayoutPasses() // LayoutBlocks() must be called after RunPasses().
+			b.runBlockLayoutPass()
 			var actual []BasicBlockID
 			for blk := b.BlockIteratorReversePostOrderBegin(); blk != nil; blk = b.BlockIteratorReversePostOrderNext() {
 				actual = append(actual, blk.(*basicBlock).id)
diff --git a/internal/engine/wazevo/ssa/pass_cfg_test.go b/internal/engine/wazevo/ssa/pass_cfg_test.go
index 014428ed..f6c48e6e 100644
--- a/internal/engine/wazevo/ssa/pass_cfg_test.go
+++ b/internal/engine/wazevo/ssa/pass_cfg_test.go
@@ -649,7 +649,7 @@ func TestBuildLoopNestingForest(t *testing.T) {
 			b := constructGraphFromEdges(tc.edges)
 			// buildLoopNestingForest requires passCalculateImmediateDominators to be done.
 			passCalculateImmediateDominators(b)
-			buildLoopNestingForest(b)
+			passBuildLoopNestingForest(b)
 
 			blocks := map[BasicBlockID]*basicBlock{}
 			for blk := b.blockIteratorBegin(); blk != nil; blk = b.blockIteratorNext() {
@@ -833,7 +833,7 @@ func TestDominatorTree(t *testing.T) {
 			b := constructGraphFromEdges(tc.edges)
 			// buildDominatorTree requires passCalculateImmediateDominators to be done.
 			passCalculateImmediateDominators(b)
-			buildDominatorTree(b)
+			passBuildDominatorTree(b)
 
 			for _, c := range tc.cases {
 				exp := b.sparseTree.findLCA(c.a, c.b)