Repository: bytecodealliance/wasmtime
Pull request #7452 (2023-11-02 04:13:30)

aarch64: Add support for `tbz` and `tbnz`
I noticed these instructions when glancing at some disassembly for other code and also noticed that Cranelift didn't have support for them. This adds a few new lowerings for conditional branches in the aarch64 backend which are for testing a bit and branching if it's zero or not zero.

<!--
Please make sure you include the following information:

- If this work has been discussed elsewhere, please include a link to that
  conversation. If it was discussed in an issue, just mention "issue #...".

- Explain why this change is needed. If the details are in an issue already,
  this can be brief.

Our development process is documented in the Wasmtime book:
https://docs.wasmtime.dev/contributing-development-process.html

Please ensure all communication follows the code of conduct:
https://github.com/bytecodealliance/wasmtime/blob/main/CODE_OF_CONDUCT.md
-->


github-actions[bot]: #### Subscribe to Label Action

cc @cfallin, @fitzgen

<details>
This issue or pull request has been labeled: "cranelift", "cranelift:area:aarch64", "isle"

Thus the following users have been cc'd because of the following labels:

* cfallin: isle
* fitzgen: isle

To subscribe or unsubscribe from this label, edit the <code>.github/subscribe-to-label.json</code> configuration file.

[Learn more.](https://github.com/bytecodealliance/subscribe-to-label-action)
</details>

Patch:
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 90c091118..314d09b15 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -833,6 +833,15 @@
         (not_taken BranchTarget)
         (kind CondBrKind))
 
+       ;; A conditional branch which tests the `bit` of `rn` and branches
+       ;; depending on `kind`.
+       (TestBitAndBranch
+        (kind TestBitAndBranchKind)
+        (taken BranchTarget)
+        (not_taken BranchTarget)
+        (rn Reg)
+        (bit u8))
+
        ;; A conditional trap: execute a `udf` if the condition is true. This is
        ;; one VCode instruction because it uses embedded control flow; it is
        ;; logically a single-in, single-out region, but needs to appear as one
@@ -1203,6 +1212,8 @@
       (enum Size32
             Size64))
 
+(type TestBitAndBranchKind (enum (Z) (NZ)))
+
 ;; Helper for calculating the `OperandSize` corresponding to a type
 (decl operand_size (Type) OperandSize)
 (rule 1 (operand_size (fits_in_32 _ty)) (OperandSize.Size32))
@@ -4079,6 +4090,21 @@
       (ConsumesFlags.ConsumesFlagsSideEffect
        (MInst.CondBr taken not_taken kind)))
 
+;; Helper for emitting `MInst.TestBitAndBranch` instructions.
+(decl test_branch (TestBitAndBranchKind BranchTarget BranchTarget Reg u8) SideEffectNoResult)
+(rule (test_branch kind taken not_taken rn bit)
+      (SideEffectNoResult.Inst (MInst.TestBitAndBranch kind taken not_taken rn bit)))
+
+;; Helper for emitting `tbnz` instructions.
+(decl tbnz (BranchTarget BranchTarget Reg u8) SideEffectNoResult)
+(rule (tbnz taken not_taken rn bit)
+      (test_branch (TestBitAndBranchKind.NZ) taken not_taken rn bit))
+
+;; Helper for emitting `tbz` instructions.
+(decl tbz (BranchTarget BranchTarget Reg u8) SideEffectNoResult)
+(rule (tbz taken not_taken rn bit)
+      (test_branch (TestBitAndBranchKind.Z) taken not_taken rn bit))
+
 ;; Helper for emitting `MInst.MovToNZCV` instructions.
 (decl mov_to_nzcv (Reg) ProducesFlags)
 (rule (mov_to_nzcv rn)
diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs
index c6394e659..f6d7f5800 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/args.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs
@@ -340,26 +340,31 @@ impl BranchTarget {
         }
     }
 
+    /// Return the target's offset, if specified, or zero if label-based.
+    pub fn as_offset14_or_zero(self) -> u32 {
+        self.as_offset_bounded(14)
+    }
+
     /// Return the target's offset, if specified, or zero if label-based.
     pub fn as_offset19_or_zero(self) -> u32 {
-        let off = match self {
-            BranchTarget::ResolvedOffset(off) => off >> 2,
-            _ => 0,
-        };
-        assert!(off <= 0x3ffff);
-        assert!(off >= -0x40000);
-        (off as u32) & 0x7ffff
+        self.as_offset_bounded(19)
     }
 
     /// Return the target's offset, if specified, or zero if label-based.
     pub fn as_offset26_or_zero(self) -> u32 {
+        self.as_offset_bounded(26)
+    }
+
+    fn as_offset_bounded(self, bits: u32) -> u32 {
         let off = match self {
             BranchTarget::ResolvedOffset(off) => off >> 2,
             _ => 0,
         };
-        assert!(off <= 0x1ffffff);
-        assert!(off >= -0x2000000);
-        (off as u32) & 0x3ffffff
+        let hi = (1 << (bits - 1)) - 1;
+        let lo = -(1 << bits - 1);
+        assert!(off <= hi);
+        assert!(off >= lo);
+        (off as u32) & ((1 << bits) - 1)
     }
 }
 
@@ -764,3 +769,15 @@ impl APIKey {
         0xd503201f | (crm << 8) | (op2 << 5)
     }
 }
+
+pub use crate::isa::aarch64::lower::isle::generated_code::TestBitAndBranchKind;
+
+impl TestBitAndBranchKind {
+    /// Complements this branch condition to act on the opposite result.
+    pub fn complement(&self) -> TestBitAndBranchKind {
+        match self {
+            TestBitAndBranchKind::Z => TestBitAndBranchKind::NZ,
+            TestBitAndBranchKind::NZ => TestBitAndBranchKind::Z,
+        }
+    }
+}
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 39a49dfdd..7549ee4a6 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -172,6 +172,27 @@ fn enc_conditional_br(
     }
 }
 
+fn enc_test_bit_and_branch(
+    kind: TestBitAndBranchKind,
+    taken: BranchTarget,
+    reg: Reg,
+    bit: u8,
+) -> u32 {
+    assert!(bit < 64);
+    let op_31 = u32::from(bit >> 5);
+    let op_23_19 = u32::from(bit & 0b11111);
+    let op_30_24 = 0b0110110
+        | match kind {
+            TestBitAndBranchKind::Z => 0,
+            TestBitAndBranchKind::NZ => 1,
+        };
+    (op_31 << 31)
+        | (op_30_24 << 24)
+        | (op_23_19 << 19)
+        | (taken.as_offset14_or_zero() << 5)
+        | machreg_to_gpr(reg)
+}
+
 fn enc_move_wide(op: MoveWideOp, rd: Writable<Reg>, imm: MoveWideConst, size: OperandSize) -> u32 {
     assert!(imm.shift <= 0b11);
     let op = match op {
@@ -3224,6 +3245,32 @@ impl MachInstEmit for Inst {
                 }
                 sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
             }
+            &Inst::TestBitAndBranch {
+                taken,
+                not_taken,
+                kind,
+                rn,
+                bit,
+            } => {
+                let rn = allocs.next(rn);
+                // Emit the conditional branch first
+                let cond_off = sink.cur_offset();
+                if let Some(l) = taken.as_label() {
+                    sink.use_label_at_offset(cond_off, l, LabelUse::Branch14);
+                    let inverted =
+                        enc_test_bit_and_branch(kind.complement(), taken, rn, bit).to_le_bytes();
+                    sink.add_cond_branch(cond_off, cond_off + 4, l, &inverted[..]);
+                }
+                sink.put4(enc_test_bit_and_branch(kind, taken, rn, bit));
+
+                // Unconditional part next.
+                let uncond_off = sink.cur_offset();
+                if let Some(l) = not_taken.as_label() {
+                    sink.use_label_at_offset(uncond_off, l, LabelUse::Branch26);
+                    sink.add_uncond_branch(uncond_off, uncond_off + 4, l);
+                }
+                sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
+            }
             &Inst::TrapIf { kind, trap_code } => {
                 let label = sink.defer_trap(trap_code, state.take_stack_map());
                 // condbr KIND, LABEL
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index bf3bd50e6..d6027c074 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -902,6 +902,9 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
             }
             CondBrKind::Cond(_) => {}
         },
+        &Inst::TestBitAndBranch { rn, .. } => {
+            collector.reg_use(rn);
+        }
         &Inst::IndirectBr { rn, .. } => {
             collector.reg_use(rn);
         }
@@ -1043,6 +1046,7 @@ impl MachInst for Inst {
             &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall,
             &Inst::Jump { .. } => MachTerminator::Uncond,
             &Inst::CondBr { .. } => MachTerminator::Cond,
+            &Inst::TestBitAndBranch { .. } => MachTerminator::Cond,
             &Inst::IndirectBr { .. } => MachTerminator::Indirect,
             &Inst::JTSequence { .. } => MachTerminator::Indirect,
             _ => MachTerminator::None,
@@ -2663,6 +2667,22 @@ impl Inst {
                     }
                 }
             }
+            &Inst::TestBitAndBranch {
+                kind,
+                ref taken,
+                ref not_taken,
+                rn,
+                bit,
+            } => {
+                let cond = match kind {
+                    TestBitAndBranchKind::Z => "z",
+                    TestBitAndBranchKind::NZ => "nz",
+                };
+                let taken = taken.pretty_print(0, allocs);
+                let not_taken = not_taken.pretty_print(0, allocs);
+                let rn = pretty_print_reg(rn, allocs);
+                format!("tb{cond} {rn}, #{bit}, {taken} ; b {not_taken}")
+            }
             &Inst::IndirectBr { rn, .. } => {
                 let rn = pretty_print_reg(rn, allocs);
                 format!("br {}", rn)
@@ -2882,6 +2902,9 @@ impl Inst {
 /// Different forms of label references for different instruction formats.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum LabelUse {
+    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
+    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
+    Branch14,
     /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
     /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
     Branch19,
@@ -2908,8 +2931,10 @@ impl MachInstLabelUse for LabelUse {
     /// Maximum PC-relative range (positive), inclusive.
     fn max_pos_range(self) -> CodeOffset {
         match self {
-            // 19-bit immediate, left-shifted by 2, for 21 bits of total range. Signed, so +2^20
-            // from zero. Likewise for two other shifted cases below.
+            // N-bit immediate, left-shifted by 2, for (N+2) bits of total
+            // range. Signed, so +2^(N+1) from zero. Likewise for two other
+            // shifted cases below.
+            LabelUse::Branch14 => (1 << 15) - 1,
             LabelUse::Branch19 => (1 << 20) - 1,
             LabelUse::Branch26 => (1 << 27) - 1,
             LabelUse::Ldr19 => (1 << 20) - 1,
@@ -2941,6 +2966,7 @@ impl MachInstLabelUse for LabelUse {
         let pc_rel = pc_rel as u32;
         let insn_word = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
         let mask = match self {
+            LabelUse::Branch14 => 0x0007ffe0, // bits 18..5 inclusive
             LabelUse::Branch19 => 0x00ffffe0, // bits 23..5 inclusive
             LabelUse::Branch26 => 0x03ffffff, // bits 25..0 inclusive
             LabelUse::Ldr19 => 0x00ffffe0,    // bits 23..5 inclusive
@@ -2955,6 +2981,7 @@ impl MachInstLabelUse for LabelUse {
             }
         };
         let pc_rel_inserted = match self {
+            LabelUse::Branch14 => (pc_rel_shifted & 0x3fff) << 5,
             LabelUse::Branch19 | LabelUse::Ldr19 => (pc_rel_shifted & 0x7ffff) << 5,
             LabelUse::Branch26 => pc_rel_shifted & 0x3ffffff,
             LabelUse::Adr21 => (pc_rel_shifted & 0x7ffff) << 5 | (pc_rel_shifted & 0x180000) << 10,
@@ -2975,8 +3002,8 @@ impl MachInstLabelUse for LabelUse {
     /// Is a veneer supported for this label reference type?
     fn supports_veneer(self) -> bool {
         match self {
-            LabelUse::Branch19 => true, // veneer is a Branch26
-            LabelUse::Branch26 => true, // veneer is a PCRel32
+            LabelUse::Branch14 | LabelUse::Branch19 => true, // veneer is a Branch26
+            LabelUse::Branch26 => true,                      // veneer is a PCRel32
             _ => false,
         }
     }
@@ -2984,7 +3011,7 @@ impl MachInstLabelUse for LabelUse {
     /// How large is the veneer, if supported?
     fn veneer_size(self) -> CodeOffset {
         match self {
-            LabelUse::Branch19 => 4,
+            LabelUse::Branch14 | LabelUse::Branch19 => 4,
             LabelUse::Branch26 => 20,
             _ => unreachable!(),
         }
@@ -3002,7 +3029,7 @@ impl MachInstLabelUse for LabelUse {
         veneer_offset: CodeOffset,
     ) -> (CodeOffset, LabelUse) {
         match self {
-            LabelUse::Branch19 => {
+            LabelUse::Branch14 | LabelUse::Branch19 => {
                 // veneer is a Branch26 (unconditional branch). Just encode directly here -- don't
                 // bother with constructing an Inst.
                 let insn_word = 0b000101 << 26;
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 4d8b29fb6..7af210286 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -2882,6 +2882,24 @@
         (with_flags_side_effect flags
          (cond_br taken not_taken (cond_br_not_zero rt))))))
 
+;; Special lowerings for `tbnz` - "Test bit and Branch if Nonzero"
+(rule 1 (lower_branch (brif (band x @ (value_type ty) (u64_from_iconst n)) _ _)
+                     (two_targets taken not_taken))
+  (if-let bit (test_and_compare_bit_const ty n))
+  (emit_side_effect (tbnz taken not_taken x bit)))
+
+;; Special lowering for `tbz` - "Test bit and Branch if Zero"
+(rule 1 (lower_branch (brif (icmp (IntCC.Equal)
+                                  (band x @ (value_type (fits_in_64 ty))
+                                        (u64_from_iconst n))
+                                  (u64_from_iconst 0)) _ _)
+                     (two_targets taken not_taken))
+  (if-let bit (test_and_compare_bit_const ty n))
+  (emit_side_effect (tbz taken not_taken x bit)))
+
+(decl pure partial test_and_compare_bit_const (Type u64) u8)
+(extern constructor test_and_compare_bit_const test_and_compare_bit_const)
+
 ;;; Rules for `jump` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower_branch (jump _) (single_target label))
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
index 0c3f151c4..1208f519c 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
@@ -812,4 +812,15 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
     fn uimm12_scaled_from_i64(&mut self, val: i64, ty: Type) -> Option<UImm12Scaled> {
         UImm12Scaled::maybe_from_i64(val, ty)
     }
+
+    fn test_and_compare_bit_const(&mut self, ty: Type, n: u64) -> Option<u8> {
+        if n.count_ones() != 1 {
+            return None;
+        }
+        let bit = n.trailing_zeros();
+        if bit >= ty.bits() {
+            return None;
+        }
+        Some(bit as u8)
+    }
 }
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 8627f2a29..0c4e1e3b5 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -1783,9 +1783,6 @@
 
 
 
-;; Generate a mask for the bit-width of the given type
-(decl pure shift_mask (Type) u64)
-(rule (shift_mask ty) (u64_sub (ty_bits (lane_type ty)) 1))
 
 ;; Helper for generating a i64 from a pair of Imm20 and Imm12 constants
 (decl i64_generate_imm (Imm20 Imm12) i64)
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 0d3a271f4..e3fa6a9f2 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -1193,7 +1193,7 @@
 
 ;; 8/16 bit types need a mask on the shift amount
 (rule 0 (lower (has_type (ty_int (ty_8_or_16 ty)) (ishl x y)))
-  (if-let mask (u64_to_imm12 (shift_mask ty)))
+  (if-let mask (u64_to_imm12 (ty_shift_mask ty)))
   (rv_sllw x (rv_andi (value_regs_get y 0) mask)))
 
 ;; Using the 32bit version of `sll` automatically masks the shift amount.
@@ -1206,12 +1206,12 @@
 
 ;; If the shift amount is known. We can mask it and encode it in the instruction.
 (rule 2 (lower (has_type (int_fits_in_32 ty) (ishl x (maybe_uextend (imm12_from_value y)))))
-  (rv_slliw x (imm12_and y (shift_mask ty))))
+  (rv_slliw x (imm12_and y (ty_shift_mask ty))))
 
 ;; We technically don't need to mask the shift amount here. The instruction
 ;; does the right thing. But it's neater when pretty printing it.
 (rule 3 (lower (has_type ty @ $I64 (ishl x (maybe_uextend (imm12_from_value y)))))
-  (rv_slli x (imm12_and y (shift_mask ty))))
+  (rv_slli x (imm12_and y (ty_shift_mask ty))))
 
 ;; With `Zba` we have a shift that zero extends the LHS argument.
 (rule 4 (lower (has_type $I64 (ishl (uextend x @ (value_type $I32)) (maybe_uextend (imm12_from_value y)))))
@@ -1253,7 +1253,7 @@
 ;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be
 ;; zero extended.
 (rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x y)))
-  (if-let mask (u64_to_imm12 (shift_mask ty)))
+  (if-let mask (u64_to_imm12 (ty_shift_mask ty)))
   (rv_srlw (zext x) (rv_andi (value_regs_get y 0) mask)))
 
 ;; Using the 32bit version of `srl` automatically masks the shift amount.
@@ -1266,7 +1266,7 @@
 
 ;; When the RHS is known we can just encode it in the instruction.
 (rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x (maybe_uextend (imm12_from_value y)))))
-  (rv_srliw (zext x) (imm12_and y (shift_mask ty))))
+  (rv_srliw (zext x) (imm12_and y (ty_shift_mask ty))))
 
 (rule 3 (lower (has_type $I32 (ushr x (maybe_uextend (imm12_from_value y)))))
   (rv_srliw x y))
@@ -1308,7 +1308,7 @@
 ;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be
 ;; zero extended.
 (rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x y)))
-  (if-let mask (u64_to_imm12 (shift_mask ty)))
+  (if-let mask (u64_to_imm12 (ty_shift_mask ty)))
   (rv_sraw (sext x) (rv_andi (value_regs_get y 0) mask)))
 
 ;; Using the 32bit version of `sra` automatically masks the shift amount.
@@ -1321,7 +1321,7 @@
 
 ;; When the RHS is known we can just encode it in the instruction.
 (rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x (maybe_uextend (imm12_from_value y)))))
-  (rv_sraiw (sext x) (imm12_and y (shift_mask ty))))
+  (rv_sraiw (sext x) (imm12_and y (ty_shift_mask ty))))
 
 (rule 3 (lower (has_type $I32 (sshr x (maybe_uextend (imm12_from_value y)))))
   (rv_sraiw x y))
diff --git a/cranelift/codegen/src/opts/bitops.isle b/cranelift/codegen/src/opts/bitops.isle
index 018db93b6..6c89e0f07 100644
--- a/cranelift/codegen/src/opts/bitops.isle
+++ b/cranelift/codegen/src/opts/bitops.isle
@@ -98,11 +98,11 @@
 ;; (x | -x) sets the sign bit to 1 if x is nonzero, and 0 if x is zero. sshr propagates
 ;; the sign bit to the rest of the value.
 (rule (simplify (sshr ty (bor ty x (ineg ty x)) (iconst ty (u64_from_imm64 shift_amt))))
-      (if-let $true (u64_eq shift_amt (u64_sub (ty_bits ty) 1)))
+      (if-let $true (u64_eq shift_amt (ty_shift_mask ty)))
       (bmask ty x))
 
 (rule (simplify (sshr ty (bor ty (ineg ty x) x) (iconst ty (u64_from_imm64 shift_amt))))
-      (if-let $true (u64_eq shift_amt (u64_sub (ty_bits ty) 1)))
+      (if-let $true (u64_eq shift_amt (ty_shift_mask ty)))
       (bmask ty x))
 
 ;; Matches any expressions that preserve "truthiness".
diff --git a/cranelift/codegen/src/opts/shifts.isle b/cranelift/codegen/src/opts/shifts.isle
index 1b33d89d8..acf53946c 100644
--- a/cranelift/codegen/src/opts/shifts.isle
+++ b/cranelift/codegen/src/opts/shifts.isle
@@ -100,7 +100,7 @@
 
 ;; ineg(ushr(x, k)) == sshr(x, k) when k == ty_bits - 1.
 (rule (simplify (ineg ty (ushr ty x sconst @ (iconst ty (u64_from_imm64 shift_amt)))))
-      (if-let $true (u64_eq shift_amt (u64_sub (ty_bits ty) 1)))
+      (if-let $true (u64_eq shift_amt (ty_shift_mask ty)))
       (sshr ty x sconst))
 
 ;; Shifts and rotates allow a different type for the shift amount, so we
@@ -144,3 +144,127 @@
 (rule (simplify (sshr ty x (iconcat _ y _))) (sshr ty x y))
 (rule (simplify (rotr ty x (iconcat _ y _))) (rotr ty x y))
 (rule (simplify (rotl ty x (iconcat _ y _))) (rotl ty x y))
+
+;; Try to combine the shift amount from multiple consecutive shifts
+;; This only works if the shift amount remains smaller than the bit
+;; width of the type.
+;;
+;; (ishl (ishl x k1) k2) == (ishl x (add k1 k2)) if shift_mask(k1) + shift_mask(k2) < ty_bits
+;; (ushr (ushr x k1) k2) == (ushr x (add k1 k2)) if shift_mask(k1) + shift_mask(k2) < ty_bits
+;; (sshr (sshr x k1) k2) == (sshr x (add k1 k2)) if shift_mask(k1) + shift_mask(k2) < ty_bits
+(rule (simplify (ishl ty
+                      (ishl ty x (iconst kty (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_lt shift_amt (ty_bits_u64 (lane_type ty))))
+      (ishl ty x (iconst_u64 kty shift_amt)))
+
+(rule (simplify (ushr ty
+                      (ushr ty x (iconst kty (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_lt shift_amt (ty_bits_u64 (lane_type ty))))
+      (ushr ty x (iconst_u64 kty shift_amt)))
+
+(rule (simplify (sshr ty
+                      (sshr ty x (iconst kty (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_lt shift_amt (ty_bits_u64 (lane_type ty))))
+      (sshr ty x (iconst_u64 kty shift_amt)))
+
+;; Simliarly, if the shift amount overflows the type, then we can turn
+;; it into a 0
+;;
+;; (ishl (ishl x k1) k2) == 0 if shift_mask(k1) + shift_mask(k2) >= ty_bits
+;; (ushr (ushr x k1) k2) == 0 if shift_mask(k1) + shift_mask(k2) >= ty_bits
+(rule (simplify (ishl ty
+                      (ishl ty x (iconst _ (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_le (ty_bits_u64 ty) shift_amt))
+      (subsume (iconst_u64 ty 0)))
+
+(rule (simplify (ushr ty
+                      (ushr ty x (iconst _ (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_le (ty_bits_u64 ty) shift_amt))
+      (subsume (iconst_u64 ty 0)))
+
+;; (rotl (rotr x y) y) == x
+;; (rotr (rotl x y) y) == x
+(rule (simplify (rotl ty (rotr ty x y) y)) (subsume x))
+(rule (simplify (rotr ty (rotl ty x y) y)) (subsume x))
+
+;; Emits an iadd for two values. If they have different types
+;; then the smaller type is zero extended to the larger type.
+(decl iadd_uextend (Value Value) Value)
+(rule 1 (iadd_uextend x @ (value_type ty) y @ (value_type ty))
+      (iadd ty x y))
+(rule 2 (iadd_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 x_ty) (ty_bits_u64 y_ty)))
+      (iadd y_ty (uextend y_ty x) y))
+(rule 3 (iadd_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 y_ty) (ty_bits_u64 x_ty)))
+      (iadd x_ty x (uextend x_ty y)))
+
+;; Emits an isub for two values. If they have different types
+;; then the smaller type is zero extended to the larger type.
+(decl isub_uextend (Value Value) Value)
+(rule 1 (isub_uextend x @ (value_type ty) y @ (value_type ty))
+      (isub ty x y))
+(rule 2 (isub_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 x_ty) (ty_bits_u64 y_ty)))
+      (isub y_ty (uextend y_ty x) y))
+(rule 3 (isub_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 y_ty) (ty_bits_u64 x_ty)))
+      (isub x_ty x (uextend x_ty y)))
+
+;; Try to group constants together so that other cprop rules can optimize them.
+;;
+;; (rotr (rotr x y) z) == (rotr x (iadd y z))
+;; (rotl (rotl x y) z) == (rotl x (iadd y z))
+;; (rotr (rotl x y) z) == (rotr x (isub y z))
+;; (rotl (rotr x y) z) == (rotl x (isub y z))
+;;
+;; if x or z are constants
+(rule (simplify (rotl ty (rotl ty x y @ (iconst _ _)) z)) (rotl ty x (iadd_uextend y z)))
+(rule (simplify (rotl ty (rotl ty x y) z @ (iconst _ _))) (rotl ty x (iadd_uextend y z)))
+(rule (simplify (rotr ty (rotr ty x y @ (iconst _ _)) z)) (rotr ty x (iadd_uextend y z)))
+(rule (simplify (rotr ty (rotr ty x y) z @ (iconst _ _))) (rotr ty x (iadd_uextend y z)))
+
+(rule (simplify (rotr ty (rotl ty x y @ (iconst _ _)) z)) (rotl ty x (isub_uextend y z)))
+(rule (simplify (rotr ty (rotl ty x y) z @ (iconst _ _))) (rotl ty x (isub_uextend y z)))
+(rule (simplify (rotl ty (rotr ty x y @ (iconst _ _)) z)) (rotr ty x (isub_uextend y z)))
+(rule (simplify (rotl ty (rotr ty x y) z @ (iconst _ _))) (rotr ty x (isub_uextend y z)))
+
+;; Similarly to the rules above, if y and z have the same type, we should emit
+;; an iadd or isub instead. In some backends this is cheaper than a rotate.
+;;
+;; If they have different types we end up in a situation where we have to insert
+;; and additional extend and that transformation is not universally beneficial.
+;;
+;; (rotr (rotr x y) z) == (rotr x (iadd y z))
+;; (rotl (rotl x y) z) == (rotl x (iadd y z))
+;; (rotr (rotl x y) z) == (rotl x (isub y z))
+;; (rotl (rotr x y) z) == (rotr x (isub y z))
+(rule (simplify (rotr ty (rotr ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotr ty x (iadd_uextend y z)))
+(rule (simplify (rotl ty (rotl ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotl ty x (iadd_uextend y z)))
+
+(rule (simplify (rotr ty (rotl ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotl ty x (isub_uextend y z)))
+(rule (simplify (rotl ty (rotr ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotr ty x (isub_uextend y z)))
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index eb03dd24a..c61876b8d 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -323,6 +323,10 @@
 (decl pure partial ty_half_width (Type) Type)
 (extern constructor ty_half_width ty_half_width)
 
+;; Generate a mask for the maximum shift amount for a given type. i.e 31 for I32.
+(decl pure ty_shift_mask (Type) u64)
+(rule (ty_shift_mask ty) (u64_sub (ty_bits (lane_type ty)) 1))
+
 ;; Compare two types for equality.
 (decl pure ty_equal (Type Type) bool)
 (extern constructor ty_equal ty_equal)
diff --git a/cranelift/codegen/src/prelude_opt.isle b/cranelift/codegen/src/prelude_opt.isle
index 099930f5b..921c7c65b 100644
--- a/cranelift/codegen/src/prelude_opt.isle
+++ b/cranelift/codegen/src/prelude_opt.isle
@@ -43,3 +43,16 @@
 ;; answer".
 (decl subsume (Value) Value)
 (extern constructor subsume subsume)
+
+;;;;; constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl iconst_u64 (Type u64) Value)
+
+;; Use a single iconst for types that fit in 64 bits.
+(rule 0 (iconst_u64 (ty_int_ref_scalar_64 ty) val)
+    (if-let $true (u64_le val (ty_umax ty)))
+    (iconst ty (imm64_masked ty val)))
+
+;; For i128 types use a iconst, but zero extend it to i128.
+(rule 1 (iconst_u64 $I128 val)
+    (uextend $I128 (iconst $I64 (imm64_masked $I64 val))))
diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs
index 03900707a..d785435b6 100644
--- a/crates/cli-flags/src/lib.rs
+++ b/crates/cli-flags/src/lib.rs
@@ -65,6 +65,14 @@ wasmtime_option_group! {
         /// Enable the pooling allocator, in place of the on-demand allocator.
         pub pooling_allocator: Option<bool>,
 
+        /// How many bytes to keep resident between instantiations for the
+        /// pooling allocator in linear memories.
+        pub pooling_memory_keep_resident: Option<usize>,
+
+        /// How many bytes to keep resident between instantiations for the
+        /// pooling allocator in tables.
+        pub pooling_table_keep_resident: Option<usize>,
+
         /// Configure attempting to initialize linear memory via a
         /// copy-on-write mapping (default: yes)
         pub memory_init_cow: Option<bool>,
@@ -500,7 +508,14 @@ impl CommonOptions {
             ["pooling-allocator" : self.opts.pooling_allocator]
             enable => {
                 if enable {
-                    config.allocation_strategy(wasmtime::InstanceAllocationStrategy::pooling());
+                    let mut cfg = wasmtime::PoolingAllocationConfig::default();
+                    if let Some(size) = self.opts.pooling_memory_keep_resident {
+                        cfg.linear_memory_keep_resident(size);
+                    }
+                    if let Some(size) = self.opts.pooling_table_keep_resident {
+                        cfg.table_keep_resident(size);
+                    }
+                    config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(cfg));
                 }
             },
             true => err,
diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs
index 5b4909b48..517b5d1b9 100644
--- a/crates/jit/src/code_memory.rs
+++ b/crates/jit/src/code_memory.rs
@@ -149,22 +149,26 @@ impl CodeMemory {
     }
 
     /// Returns a reference to the underlying `MmapVec` this memory owns.
+    #[inline]
     pub fn mmap(&self) -> &MmapVec {
         &self.mmap
     }
 
     /// Returns the contents of the text section of the ELF executable this
     /// represents.
+    #[inline]
     pub fn text(&self) -> &[u8] {
         &self.mmap[self.text.clone()]
     }
 
     /// Returns the contents of the `ELF_WASMTIME_DWARF` section.
+    #[inline]
     pub fn dwarf(&self) -> &[u8] {
         &self.mmap[self.dwarf.clone()]
     }
 
     /// Returns the data in the `ELF_NAME_DATA` section.
+    #[inline]
     pub fn func_name_data(&self) -> &[u8] {
         &self.mmap[self.func_name_data.clone()]
     }
@@ -174,24 +178,28 @@ impl CodeMemory {
     ///
     /// This is used for initialization of memories and all data ranges stored
     /// in a `Module` are relative to the slice returned here.
+    #[inline]
     pub fn wasm_data(&self) -> &[u8] {
         &self.mmap[self.wasm_data.clone()]
     }
 
     /// Returns the encoded address map section used to pass to
     /// `wasmtime_environ::lookup_file_pos`.
+    #[inline]
     pub fn address_map_data(&self) -> &[u8] {
         &self.mmap[self.address_map_data.clone()]
     }
 
     /// Returns the contents of the `ELF_WASMTIME_INFO` section, or an empty
     /// slice if it wasn't found.
+    #[inline]
     pub fn wasmtime_info(&self) -> &[u8] {
         &self.mmap[self.info_data.clone()]
     }
 
     /// Returns the contents of the `ELF_WASMTIME_TRAPS` section, or an empty
     /// slice if it wasn't found.
+    #[inline]
     pub fn trap_data(&self) -> &[u8] {
         &self.mmap[self.trap_data.clone()]
     }
diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs
index 4f9f4c64d..341cd4dee 100644
--- a/crates/runtime/src/mmap.rs
+++ b/crates/runtime/src/mmap.rs
@@ -146,11 +146,13 @@ impl Mmap {
     }
 
     /// Return the allocated memory as a pointer to u8.
+    #[inline]
     pub fn as_ptr(&self) -> *const u8 {
         self.sys.as_ptr()
     }
 
     /// Return the allocated memory as a mutable pointer to u8.
+    #[inline]
     pub fn as_mut_ptr(&mut self) -> *mut u8 {
         self.sys.as_mut_ptr()
     }
@@ -159,6 +161,7 @@ impl Mmap {
     ///
     /// This is the byte length of this entire mapping which includes both
     /// addressible and non-addressible memory.
+    #[inline]
     pub fn len(&self) -> usize {
         self.sys.len()
     }
diff --git a/crates/runtime/src/mmap/unix.rs b/crates/runtime/src/mmap/unix.rs
index 3227ec3a0..97de914da 100644
--- a/crates/runtime/src/mmap/unix.rs
+++ b/crates/runtime/src/mmap/unix.rs
@@ -84,14 +84,17 @@ impl Mmap {
         Ok(())
     }
 
+    #[inline]
     pub fn as_ptr(&self) -> *const u8 {
         self.memory.as_ptr() as *const u8
     }
 
+    #[inline]
     pub fn as_mut_ptr(&mut self) -> *mut u8 {
         self.memory.as_ptr().cast()
     }
 
+    #[inline]
     pub fn len(&self) -> usize {
         unsafe { (*self.memory.as_ptr()).len() }
     }
diff --git a/crates/runtime/src/mmap/windows.rs b/crates/runtime/src/mmap/windows.rs
index 5ca208b33..4a8396c44 100644
--- a/crates/runtime/src/mmap/windows.rs
+++ b/crates/runtime/src/mmap/windows.rs
@@ -147,14 +147,17 @@ impl Mmap {
         Ok(())
     }
 
+    #[inline]
     pub fn as_ptr(&self) -> *const u8 {
         self.memory.as_ptr() as *const u8
     }
 
+    #[inline]
     pub fn as_mut_ptr(&mut self) -> *mut u8 {
         self.memory.as_ptr().cast()
     }
 
+    #[inline]
     pub fn len(&self) -> usize {
         unsafe { (*self.memory.as_ptr()).len() }
     }


Test patch:
diff --git a/cranelift/filetests/filetests/egraph/shifts.clif b/cranelift/filetests/filetests/egraph/shifts.clif
index 7852ada12..dbe984c84 100644
--- a/cranelift/filetests/filetests/egraph/shifts.clif
+++ b/cranelift/filetests/filetests/egraph/shifts.clif
@@ -524,3 +524,237 @@ block0(v0: i8, v1: i16, v2: i16):
 
 ; check: v5 = rotl v0, v1
 ; check: return v5
+
+function %ishl_prop_overflow(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 7
+    v2 = iconst.i8 7
+    v3 = ishl.i8 v0, v1
+    v4 = ishl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 0
+; check: return v5  ; v5 = 0
+
+function %ishl_prop_type_diff(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i16 7
+    v2 = iconst.i16 7
+    v3 = ishl.i8 v0, v1
+    v4 = ishl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 0
+; check: return v5  ; v5 = 0
+
+function %ushr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = ushr.i8 v0, v1
+    v4 = ushr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 2
+; check: v6 = ushr v0, v5  ; v5 = 2
+; check: return v6
+
+function %ushr_prop_overflow(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 7
+    v2 = iconst.i8 7
+    v3 = ushr.i8 v0, v1
+    v4 = ushr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 0
+; check: return v5  ; v5 = 0
+
+
+function %sshr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = sshr.i8 v0, v1
+    v4 = sshr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 2
+; check: v6 = sshr v0, v5  ; v5 = 2
+; check: return v6
+
+function %i128_ushr_overflow_becomes_0(i128) -> i128 {
+block0(v0: i128):
+    v1 = iconst.i16 200
+    v2 = ushr v0, v1
+    v3 = ushr v2, v1
+    return v3
+}
+
+; check: v4 = iconst.i64 0
+; nextln: v5 = uextend.i128 v4  ; v4 = 0
+; nextln: return v5
+
+
+function %i128_ishl_overflow_becomes_0(i128) -> i128 {
+block0(v0: i128):
+    v1 = iconst.i16 200
+    v2 = ishl v0, v1
+    v3 = ishl v2, v1
+    return v3
+}
+
+; check: v4 = iconst.i64 0
+; nextln: v5 = uextend.i128 v4  ; v4 = 0
+; nextln: return v5
+
+function %simd_shift_does_not_panic(i8x16) -> i8x16 {
+block0(v0: i8x16):
+    v1 = iconst.i64 0
+    v2 = ushr v0, v1
+    v3 = ushr v2, v1
+    return v3
+}
+
+; check: v1 = iconst.i64 0
+; nextln: v2 = ushr v0, v1  ; v1 = 0
+; check: return v2
+
+function %merges_shift_amount_based_on_lane_type(i8x16) -> i8x16 {
+block0(v0: i8x16):
+    v1 = iconst.i16 30
+    v2 = iconst.i16 3
+
+    v3 = sshr v0, v1
+    v4 = sshr v3, v2
+    v5 = sshr v4, v2
+    return v5
+}
+
+; check: v1 = iconst.i16 30
+; nextln: v3 = sshr v0, v1  ; v1 = 30
+; nextln: v6 = iconst.i16 6
+; nextln: v7 = sshr v3, v6  ; v6 = 6
+; check: return v7
+
+
+function %rotr_rotr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = rotr.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v11 = iconst.i8 2
+; check: v13 = rotr v0, v11  ; v11 = 2
+; check: return v13
+
+function %rotr_rotr_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotr.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iadd v1, v2
+; check: v6 = rotr v0, v5
+; check: return v6
+
+function %rotl_rotl_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = rotl.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v11 = iconst.i8 2
+; check: v13 = rotl v0, v11  ; v11 = 2
+; check: return v13
+
+function %rotl_rotl_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotl.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iadd v1, v2
+; check: v6 = rotl v0, v5
+; check: return v6
+
+
+function %rotl_rotr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 2
+    v2 = iconst.i8 1
+    v3 = rotr.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v2 = iconst.i8 1
+; check: v21 = rotr v0, v2  ; v2 = 1
+; check: return v21
+
+function %rotl_rotr_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotr.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = isub v1, v2
+; check: v6 = rotr v0, v5
+; check: return v6
+
+function %rotl_rotr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 2
+    v2 = iconst.i8 1
+    v3 = rotl.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v2 = iconst.i8 1
+; check: v21 = rotl v0, v2  ; v2 = 1
+; check: return v21
+
+function %rotl_rotr_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotl.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = isub v1, v2
+; check: v6 = rotl v0, v5
+; check: return v6
+
+function %rotl_rotr_subsume(i8, i8) -> i8 {
+block0(v0: i8, v1: i8):
+    v2 = rotl.i8 v0, v1
+    v3 = rotr.i8 v2, v1
+    return v3
+}
+
+; check: return v0
+
+
+function %rotr_rotl_subsume(i8, i8) -> i8 {
+block0(v0: i8, v1: i8):
+    v2 = rotr.i8 v0, v1
+    v3 = rotl.i8 v2, v1
+    return v3
+}
+
+; check: return v0
diff --git a/cranelift/filetests/filetests/isa/aarch64/condbr.clif b/cranelift/filetests/filetests/isa/aarch64/condbr.clif
index f0661556d..13d59de14 100644
--- a/cranelift/filetests/filetests/isa/aarch64/condbr.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/condbr.clif
@@ -733,3 +733,183 @@ block1:
 ; block1: ; offset 0x1c
 ;   ret
 
+function %tbnz_i8(i8) {
+block0(v0: i8):
+  v1 = band_imm v0, 0x10
+  brif v1, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbnz x0, #4, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbnz w0, #4, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+function %tbz_i16(i16) {
+block0(v0: i16):
+  v1 = band_imm v0, 0x1000
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbz x0, #12, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbz w0, #0xc, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+function %tbnz_i32(i32) {
+block0(v0: i32):
+  v1 = band_imm v0, 0x10000
+  brif v1, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbnz x0, #16, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbnz w0, #0x10, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+
+function %tbz_i64(i64) {
+block0(v0: i64):
+  v1 = band_imm v0, 0x1_00000000
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbz x0, #32, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbz x0, #0x20, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+function %not_tbz1(i8) {
+block0(v0: i8):
+  v1 = band_imm v0, 0x100
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   movz w4, #0
+;   and w4, w0, w4
+;   uxtb w4, w4
+;   subs wzr, w4, #0
+;   b.eq label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   mov w4, #0
+;   and w4, w0, w4
+;   uxtb w4, w4
+;   cmp w4, #0
+;   b.eq #0x18
+; block1: ; offset 0x14
+;   ret
+; block2: ; offset 0x18
+;   ret
+
+function %not_tbz2(i8) {
+block0(v0: i8):
+  v1 = band_imm v0, 0x3
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   and w3, w0, #3
+;   uxtb w3, w3
+;   subs wzr, w3, #0
+;   b.eq label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   and w3, w0, #3
+;   uxtb w3, w3
+;   cmp w3, #0
+;   b.eq #0x14
+; block1: ; offset 0x10
+;   ret
+; block2: ; offset 0x14
+;   ret