Repository: bytecodealliance/wasmtime
Pull request #7462 (2023-11-02 17:45:02)

winch: Add spec tests for `local_{get,set}`
This change is a follow up to https://github.com/bytecodealliance/wasmtime/pull/7443; after it landed I realized that Winch doesn't include spec tests for local.get and local.set.

Those tests uncovered a bug on the handling of the constant pool: given Winch's singlepass nature, there's very little room know all the constants ahead of time and to register them all at once at emission time; instead they are emitted when they are needed by an instruction. Even though Cranelift's machinery is capable of deuplicated constants in the pool, `register_constant` assumes and checks that each constat should only be pushed once. In Winch's case, since we emit as we go, we need to carefully check if the constant is one was not emitted before, and if that's the case, register it. Else we break the invariant that each constant should only be registered once.

<!--
Please make sure you include the following information:

- If this work has been discussed elsewhere, please include a link to that
  conversation. If it was discussed in an issue, just mention "issue #...".

- Explain why this change is needed. If the details are in an issue already,
  this can be brief.

Our development process is documented in the Wasmtime book:
https://docs.wasmtime.dev/contributing-development-process.html

Please ensure all communication follows the code of conduct:
https://github.com/bytecodealliance/wasmtime/blob/main/CODE_OF_CONDUCT.md
-->



Patch:
diff --git a/Cargo.toml b/Cargo.toml
index b9f539873..513105811 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -343,7 +343,7 @@ component-model = [
   "wasmtime-wast/component-model",
   "wasmtime-cli-flags/component-model"
 ]
-wat = ["dep:wat"]
+wat = ["dep:wat", "wasmtime/wat"]
 cache = ["dep:wasmtime-cache", "wasmtime-cli-flags/cache"]
 parallel-compilation = ["wasmtime-cli-flags/parallel-compilation"]
 logging = ["wasmtime-cli-flags/logging"]
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 90c091118..314d09b15 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -833,6 +833,15 @@
         (not_taken BranchTarget)
         (kind CondBrKind))
 
+       ;; A conditional branch which tests the `bit` of `rn` and branches
+       ;; depending on `kind`.
+       (TestBitAndBranch
+        (kind TestBitAndBranchKind)
+        (taken BranchTarget)
+        (not_taken BranchTarget)
+        (rn Reg)
+        (bit u8))
+
        ;; A conditional trap: execute a `udf` if the condition is true. This is
        ;; one VCode instruction because it uses embedded control flow; it is
        ;; logically a single-in, single-out region, but needs to appear as one
@@ -1203,6 +1212,8 @@
       (enum Size32
             Size64))
 
+(type TestBitAndBranchKind (enum (Z) (NZ)))
+
 ;; Helper for calculating the `OperandSize` corresponding to a type
 (decl operand_size (Type) OperandSize)
 (rule 1 (operand_size (fits_in_32 _ty)) (OperandSize.Size32))
@@ -4079,6 +4090,21 @@
       (ConsumesFlags.ConsumesFlagsSideEffect
        (MInst.CondBr taken not_taken kind)))
 
+;; Helper for emitting `MInst.TestBitAndBranch` instructions.
+(decl test_branch (TestBitAndBranchKind BranchTarget BranchTarget Reg u8) SideEffectNoResult)
+(rule (test_branch kind taken not_taken rn bit)
+      (SideEffectNoResult.Inst (MInst.TestBitAndBranch kind taken not_taken rn bit)))
+
+;; Helper for emitting `tbnz` instructions.
+(decl tbnz (BranchTarget BranchTarget Reg u8) SideEffectNoResult)
+(rule (tbnz taken not_taken rn bit)
+      (test_branch (TestBitAndBranchKind.NZ) taken not_taken rn bit))
+
+;; Helper for emitting `tbz` instructions.
+(decl tbz (BranchTarget BranchTarget Reg u8) SideEffectNoResult)
+(rule (tbz taken not_taken rn bit)
+      (test_branch (TestBitAndBranchKind.Z) taken not_taken rn bit))
+
 ;; Helper for emitting `MInst.MovToNZCV` instructions.
 (decl mov_to_nzcv (Reg) ProducesFlags)
 (rule (mov_to_nzcv rn)
diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs
index c6394e659..f6d7f5800 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/args.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs
@@ -340,26 +340,31 @@ impl BranchTarget {
         }
     }
 
+    /// Return the target's offset, if specified, or zero if label-based.
+    pub fn as_offset14_or_zero(self) -> u32 {
+        self.as_offset_bounded(14)
+    }
+
     /// Return the target's offset, if specified, or zero if label-based.
     pub fn as_offset19_or_zero(self) -> u32 {
-        let off = match self {
-            BranchTarget::ResolvedOffset(off) => off >> 2,
-            _ => 0,
-        };
-        assert!(off <= 0x3ffff);
-        assert!(off >= -0x40000);
-        (off as u32) & 0x7ffff
+        self.as_offset_bounded(19)
     }
 
     /// Return the target's offset, if specified, or zero if label-based.
     pub fn as_offset26_or_zero(self) -> u32 {
+        self.as_offset_bounded(26)
+    }
+
+    fn as_offset_bounded(self, bits: u32) -> u32 {
         let off = match self {
             BranchTarget::ResolvedOffset(off) => off >> 2,
             _ => 0,
         };
-        assert!(off <= 0x1ffffff);
-        assert!(off >= -0x2000000);
-        (off as u32) & 0x3ffffff
+        let hi = (1 << (bits - 1)) - 1;
+        let lo = -(1 << bits - 1);
+        assert!(off <= hi);
+        assert!(off >= lo);
+        (off as u32) & ((1 << bits) - 1)
     }
 }
 
@@ -764,3 +769,15 @@ impl APIKey {
         0xd503201f | (crm << 8) | (op2 << 5)
     }
 }
+
+pub use crate::isa::aarch64::lower::isle::generated_code::TestBitAndBranchKind;
+
+impl TestBitAndBranchKind {
+    /// Complements this branch condition to act on the opposite result.
+    pub fn complement(&self) -> TestBitAndBranchKind {
+        match self {
+            TestBitAndBranchKind::Z => TestBitAndBranchKind::NZ,
+            TestBitAndBranchKind::NZ => TestBitAndBranchKind::Z,
+        }
+    }
+}
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 39a49dfdd..7549ee4a6 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -172,6 +172,27 @@ fn enc_conditional_br(
     }
 }
 
+fn enc_test_bit_and_branch(
+    kind: TestBitAndBranchKind,
+    taken: BranchTarget,
+    reg: Reg,
+    bit: u8,
+) -> u32 {
+    assert!(bit < 64);
+    let op_31 = u32::from(bit >> 5);
+    let op_23_19 = u32::from(bit & 0b11111);
+    let op_30_24 = 0b0110110
+        | match kind {
+            TestBitAndBranchKind::Z => 0,
+            TestBitAndBranchKind::NZ => 1,
+        };
+    (op_31 << 31)
+        | (op_30_24 << 24)
+        | (op_23_19 << 19)
+        | (taken.as_offset14_or_zero() << 5)
+        | machreg_to_gpr(reg)
+}
+
 fn enc_move_wide(op: MoveWideOp, rd: Writable<Reg>, imm: MoveWideConst, size: OperandSize) -> u32 {
     assert!(imm.shift <= 0b11);
     let op = match op {
@@ -3224,6 +3245,32 @@ impl MachInstEmit for Inst {
                 }
                 sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
             }
+            &Inst::TestBitAndBranch {
+                taken,
+                not_taken,
+                kind,
+                rn,
+                bit,
+            } => {
+                let rn = allocs.next(rn);
+                // Emit the conditional branch first
+                let cond_off = sink.cur_offset();
+                if let Some(l) = taken.as_label() {
+                    sink.use_label_at_offset(cond_off, l, LabelUse::Branch14);
+                    let inverted =
+                        enc_test_bit_and_branch(kind.complement(), taken, rn, bit).to_le_bytes();
+                    sink.add_cond_branch(cond_off, cond_off + 4, l, &inverted[..]);
+                }
+                sink.put4(enc_test_bit_and_branch(kind, taken, rn, bit));
+
+                // Unconditional part next.
+                let uncond_off = sink.cur_offset();
+                if let Some(l) = not_taken.as_label() {
+                    sink.use_label_at_offset(uncond_off, l, LabelUse::Branch26);
+                    sink.add_uncond_branch(uncond_off, uncond_off + 4, l);
+                }
+                sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
+            }
             &Inst::TrapIf { kind, trap_code } => {
                 let label = sink.defer_trap(trap_code, state.take_stack_map());
                 // condbr KIND, LABEL
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index bf3bd50e6..d6027c074 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -902,6 +902,9 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
             }
             CondBrKind::Cond(_) => {}
         },
+        &Inst::TestBitAndBranch { rn, .. } => {
+            collector.reg_use(rn);
+        }
         &Inst::IndirectBr { rn, .. } => {
             collector.reg_use(rn);
         }
@@ -1043,6 +1046,7 @@ impl MachInst for Inst {
             &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall,
             &Inst::Jump { .. } => MachTerminator::Uncond,
             &Inst::CondBr { .. } => MachTerminator::Cond,
+            &Inst::TestBitAndBranch { .. } => MachTerminator::Cond,
             &Inst::IndirectBr { .. } => MachTerminator::Indirect,
             &Inst::JTSequence { .. } => MachTerminator::Indirect,
             _ => MachTerminator::None,
@@ -2663,6 +2667,22 @@ impl Inst {
                     }
                 }
             }
+            &Inst::TestBitAndBranch {
+                kind,
+                ref taken,
+                ref not_taken,
+                rn,
+                bit,
+            } => {
+                let cond = match kind {
+                    TestBitAndBranchKind::Z => "z",
+                    TestBitAndBranchKind::NZ => "nz",
+                };
+                let taken = taken.pretty_print(0, allocs);
+                let not_taken = not_taken.pretty_print(0, allocs);
+                let rn = pretty_print_reg(rn, allocs);
+                format!("tb{cond} {rn}, #{bit}, {taken} ; b {not_taken}")
+            }
             &Inst::IndirectBr { rn, .. } => {
                 let rn = pretty_print_reg(rn, allocs);
                 format!("br {}", rn)
@@ -2882,6 +2902,9 @@ impl Inst {
 /// Different forms of label references for different instruction formats.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum LabelUse {
+    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
+    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
+    Branch14,
     /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
     /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
     Branch19,
@@ -2908,8 +2931,10 @@ impl MachInstLabelUse for LabelUse {
     /// Maximum PC-relative range (positive), inclusive.
     fn max_pos_range(self) -> CodeOffset {
         match self {
-            // 19-bit immediate, left-shifted by 2, for 21 bits of total range. Signed, so +2^20
-            // from zero. Likewise for two other shifted cases below.
+            // N-bit immediate, left-shifted by 2, for (N+2) bits of total
+            // range. Signed, so +2^(N+1) from zero. Likewise for two other
+            // shifted cases below.
+            LabelUse::Branch14 => (1 << 15) - 1,
             LabelUse::Branch19 => (1 << 20) - 1,
             LabelUse::Branch26 => (1 << 27) - 1,
             LabelUse::Ldr19 => (1 << 20) - 1,
@@ -2941,6 +2966,7 @@ impl MachInstLabelUse for LabelUse {
         let pc_rel = pc_rel as u32;
         let insn_word = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
         let mask = match self {
+            LabelUse::Branch14 => 0x0007ffe0, // bits 18..5 inclusive
             LabelUse::Branch19 => 0x00ffffe0, // bits 23..5 inclusive
             LabelUse::Branch26 => 0x03ffffff, // bits 25..0 inclusive
             LabelUse::Ldr19 => 0x00ffffe0,    // bits 23..5 inclusive
@@ -2955,6 +2981,7 @@ impl MachInstLabelUse for LabelUse {
             }
         };
         let pc_rel_inserted = match self {
+            LabelUse::Branch14 => (pc_rel_shifted & 0x3fff) << 5,
             LabelUse::Branch19 | LabelUse::Ldr19 => (pc_rel_shifted & 0x7ffff) << 5,
             LabelUse::Branch26 => pc_rel_shifted & 0x3ffffff,
             LabelUse::Adr21 => (pc_rel_shifted & 0x7ffff) << 5 | (pc_rel_shifted & 0x180000) << 10,
@@ -2975,8 +3002,8 @@ impl MachInstLabelUse for LabelUse {
     /// Is a veneer supported for this label reference type?
     fn supports_veneer(self) -> bool {
         match self {
-            LabelUse::Branch19 => true, // veneer is a Branch26
-            LabelUse::Branch26 => true, // veneer is a PCRel32
+            LabelUse::Branch14 | LabelUse::Branch19 => true, // veneer is a Branch26
+            LabelUse::Branch26 => true,                      // veneer is a PCRel32
             _ => false,
         }
     }
@@ -2984,7 +3011,7 @@ impl MachInstLabelUse for LabelUse {
     /// How large is the veneer, if supported?
     fn veneer_size(self) -> CodeOffset {
         match self {
-            LabelUse::Branch19 => 4,
+            LabelUse::Branch14 | LabelUse::Branch19 => 4,
             LabelUse::Branch26 => 20,
             _ => unreachable!(),
         }
@@ -3002,7 +3029,7 @@ impl MachInstLabelUse for LabelUse {
         veneer_offset: CodeOffset,
     ) -> (CodeOffset, LabelUse) {
         match self {
-            LabelUse::Branch19 => {
+            LabelUse::Branch14 | LabelUse::Branch19 => {
                 // veneer is a Branch26 (unconditional branch). Just encode directly here -- don't
                 // bother with constructing an Inst.
                 let insn_word = 0b000101 << 26;
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 4d8b29fb6..7af210286 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -2882,6 +2882,24 @@
         (with_flags_side_effect flags
          (cond_br taken not_taken (cond_br_not_zero rt))))))
 
+;; Special lowerings for `tbnz` - "Test bit and Branch if Nonzero"
+(rule 1 (lower_branch (brif (band x @ (value_type ty) (u64_from_iconst n)) _ _)
+                     (two_targets taken not_taken))
+  (if-let bit (test_and_compare_bit_const ty n))
+  (emit_side_effect (tbnz taken not_taken x bit)))
+
+;; Special lowering for `tbz` - "Test bit and Branch if Zero"
+(rule 1 (lower_branch (brif (icmp (IntCC.Equal)
+                                  (band x @ (value_type (fits_in_64 ty))
+                                        (u64_from_iconst n))
+                                  (u64_from_iconst 0)) _ _)
+                     (two_targets taken not_taken))
+  (if-let bit (test_and_compare_bit_const ty n))
+  (emit_side_effect (tbz taken not_taken x bit)))
+
+(decl pure partial test_and_compare_bit_const (Type u64) u8)
+(extern constructor test_and_compare_bit_const test_and_compare_bit_const)
+
 ;;; Rules for `jump` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower_branch (jump _) (single_target label))
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
index 0c3f151c4..1208f519c 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
@@ -812,4 +812,15 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
     fn uimm12_scaled_from_i64(&mut self, val: i64, ty: Type) -> Option<UImm12Scaled> {
         UImm12Scaled::maybe_from_i64(val, ty)
     }
+
+    fn test_and_compare_bit_const(&mut self, ty: Type, n: u64) -> Option<u8> {
+        if n.count_ones() != 1 {
+            return None;
+        }
+        let bit = n.trailing_zeros();
+        if bit >= ty.bits() {
+            return None;
+        }
+        Some(bit as u8)
+    }
 }
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 8627f2a29..0c4e1e3b5 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -1783,9 +1783,6 @@
 
 
 
-;; Generate a mask for the bit-width of the given type
-(decl pure shift_mask (Type) u64)
-(rule (shift_mask ty) (u64_sub (ty_bits (lane_type ty)) 1))
 
 ;; Helper for generating a i64 from a pair of Imm20 and Imm12 constants
 (decl i64_generate_imm (Imm20 Imm12) i64)
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 0d3a271f4..e3fa6a9f2 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -1193,7 +1193,7 @@
 
 ;; 8/16 bit types need a mask on the shift amount
 (rule 0 (lower (has_type (ty_int (ty_8_or_16 ty)) (ishl x y)))
-  (if-let mask (u64_to_imm12 (shift_mask ty)))
+  (if-let mask (u64_to_imm12 (ty_shift_mask ty)))
   (rv_sllw x (rv_andi (value_regs_get y 0) mask)))
 
 ;; Using the 32bit version of `sll` automatically masks the shift amount.
@@ -1206,12 +1206,12 @@
 
 ;; If the shift amount is known. We can mask it and encode it in the instruction.
 (rule 2 (lower (has_type (int_fits_in_32 ty) (ishl x (maybe_uextend (imm12_from_value y)))))
-  (rv_slliw x (imm12_and y (shift_mask ty))))
+  (rv_slliw x (imm12_and y (ty_shift_mask ty))))
 
 ;; We technically don't need to mask the shift amount here. The instruction
 ;; does the right thing. But it's neater when pretty printing it.
 (rule 3 (lower (has_type ty @ $I64 (ishl x (maybe_uextend (imm12_from_value y)))))
-  (rv_slli x (imm12_and y (shift_mask ty))))
+  (rv_slli x (imm12_and y (ty_shift_mask ty))))
 
 ;; With `Zba` we have a shift that zero extends the LHS argument.
 (rule 4 (lower (has_type $I64 (ishl (uextend x @ (value_type $I32)) (maybe_uextend (imm12_from_value y)))))
@@ -1253,7 +1253,7 @@
 ;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be
 ;; zero extended.
 (rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x y)))
-  (if-let mask (u64_to_imm12 (shift_mask ty)))
+  (if-let mask (u64_to_imm12 (ty_shift_mask ty)))
   (rv_srlw (zext x) (rv_andi (value_regs_get y 0) mask)))
 
 ;; Using the 32bit version of `srl` automatically masks the shift amount.
@@ -1266,7 +1266,7 @@
 
 ;; When the RHS is known we can just encode it in the instruction.
 (rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x (maybe_uextend (imm12_from_value y)))))
-  (rv_srliw (zext x) (imm12_and y (shift_mask ty))))
+  (rv_srliw (zext x) (imm12_and y (ty_shift_mask ty))))
 
 (rule 3 (lower (has_type $I32 (ushr x (maybe_uextend (imm12_from_value y)))))
   (rv_srliw x y))
@@ -1308,7 +1308,7 @@
 ;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be
 ;; zero extended.
 (rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x y)))
-  (if-let mask (u64_to_imm12 (shift_mask ty)))
+  (if-let mask (u64_to_imm12 (ty_shift_mask ty)))
   (rv_sraw (sext x) (rv_andi (value_regs_get y 0) mask)))
 
 ;; Using the 32bit version of `sra` automatically masks the shift amount.
@@ -1321,7 +1321,7 @@
 
 ;; When the RHS is known we can just encode it in the instruction.
 (rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x (maybe_uextend (imm12_from_value y)))))
-  (rv_sraiw (sext x) (imm12_and y (shift_mask ty))))
+  (rv_sraiw (sext x) (imm12_and y (ty_shift_mask ty))))
 
 (rule 3 (lower (has_type $I32 (sshr x (maybe_uextend (imm12_from_value y)))))
   (rv_sraiw x y))
diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs
index c743fcd20..90bb2a286 100644
--- a/cranelift/codegen/src/machinst/vcode.rs
+++ b/cranelift/codegen/src/machinst/vcode.rs
@@ -1738,6 +1738,15 @@ impl VCodeConstants {
     pub fn get(&self, c: VCodeConstant) -> &VCodeConstantData {
         &self.constants[c]
     }
+
+    /// Checks if the given [VCodeConstantData] is registered as
+    /// used by the pool.
+    pub fn pool_uses(&self, constant: &VCodeConstantData) -> bool {
+        match constant {
+            VCodeConstantData::Pool(c, _) => self.pool_uses.contains_key(c),
+            _ => false,
+        }
+    }
 }
 
 /// A use of a constant by one or more VCode instructions; see [VCodeConstants].
diff --git a/cranelift/codegen/src/opts/bitops.isle b/cranelift/codegen/src/opts/bitops.isle
index 018db93b6..6c89e0f07 100644
--- a/cranelift/codegen/src/opts/bitops.isle
+++ b/cranelift/codegen/src/opts/bitops.isle
@@ -98,11 +98,11 @@
 ;; (x | -x) sets the sign bit to 1 if x is nonzero, and 0 if x is zero. sshr propagates
 ;; the sign bit to the rest of the value.
 (rule (simplify (sshr ty (bor ty x (ineg ty x)) (iconst ty (u64_from_imm64 shift_amt))))
-      (if-let $true (u64_eq shift_amt (u64_sub (ty_bits ty) 1)))
+      (if-let $true (u64_eq shift_amt (ty_shift_mask ty)))
       (bmask ty x))
 
 (rule (simplify (sshr ty (bor ty (ineg ty x) x) (iconst ty (u64_from_imm64 shift_amt))))
-      (if-let $true (u64_eq shift_amt (u64_sub (ty_bits ty) 1)))
+      (if-let $true (u64_eq shift_amt (ty_shift_mask ty)))
       (bmask ty x))
 
 ;; Matches any expressions that preserve "truthiness".
diff --git a/cranelift/codegen/src/opts/shifts.isle b/cranelift/codegen/src/opts/shifts.isle
index 1b33d89d8..acf53946c 100644
--- a/cranelift/codegen/src/opts/shifts.isle
+++ b/cranelift/codegen/src/opts/shifts.isle
@@ -100,7 +100,7 @@
 
 ;; ineg(ushr(x, k)) == sshr(x, k) when k == ty_bits - 1.
 (rule (simplify (ineg ty (ushr ty x sconst @ (iconst ty (u64_from_imm64 shift_amt)))))
-      (if-let $true (u64_eq shift_amt (u64_sub (ty_bits ty) 1)))
+      (if-let $true (u64_eq shift_amt (ty_shift_mask ty)))
       (sshr ty x sconst))
 
 ;; Shifts and rotates allow a different type for the shift amount, so we
@@ -144,3 +144,127 @@
 (rule (simplify (sshr ty x (iconcat _ y _))) (sshr ty x y))
 (rule (simplify (rotr ty x (iconcat _ y _))) (rotr ty x y))
 (rule (simplify (rotl ty x (iconcat _ y _))) (rotl ty x y))
+
+;; Try to combine the shift amount from multiple consecutive shifts
+;; This only works if the shift amount remains smaller than the bit
+;; width of the type.
+;;
+;; (ishl (ishl x k1) k2) == (ishl x (add k1 k2)) if shift_mask(k1) + shift_mask(k2) < ty_bits
+;; (ushr (ushr x k1) k2) == (ushr x (add k1 k2)) if shift_mask(k1) + shift_mask(k2) < ty_bits
+;; (sshr (sshr x k1) k2) == (sshr x (add k1 k2)) if shift_mask(k1) + shift_mask(k2) < ty_bits
+(rule (simplify (ishl ty
+                      (ishl ty x (iconst kty (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_lt shift_amt (ty_bits_u64 (lane_type ty))))
+      (ishl ty x (iconst_u64 kty shift_amt)))
+
+(rule (simplify (ushr ty
+                      (ushr ty x (iconst kty (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_lt shift_amt (ty_bits_u64 (lane_type ty))))
+      (ushr ty x (iconst_u64 kty shift_amt)))
+
+(rule (simplify (sshr ty
+                      (sshr ty x (iconst kty (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_lt shift_amt (ty_bits_u64 (lane_type ty))))
+      (sshr ty x (iconst_u64 kty shift_amt)))
+
+;; Simliarly, if the shift amount overflows the type, then we can turn
+;; it into a 0
+;;
+;; (ishl (ishl x k1) k2) == 0 if shift_mask(k1) + shift_mask(k2) >= ty_bits
+;; (ushr (ushr x k1) k2) == 0 if shift_mask(k1) + shift_mask(k2) >= ty_bits
+(rule (simplify (ishl ty
+                      (ishl ty x (iconst _ (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_le (ty_bits_u64 ty) shift_amt))
+      (subsume (iconst_u64 ty 0)))
+
+(rule (simplify (ushr ty
+                      (ushr ty x (iconst _ (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_le (ty_bits_u64 ty) shift_amt))
+      (subsume (iconst_u64 ty 0)))
+
+;; (rotl (rotr x y) y) == x
+;; (rotr (rotl x y) y) == x
+(rule (simplify (rotl ty (rotr ty x y) y)) (subsume x))
+(rule (simplify (rotr ty (rotl ty x y) y)) (subsume x))
+
+;; Emits an iadd for two values. If they have different types
+;; then the smaller type is zero extended to the larger type.
+(decl iadd_uextend (Value Value) Value)
+(rule 1 (iadd_uextend x @ (value_type ty) y @ (value_type ty))
+      (iadd ty x y))
+(rule 2 (iadd_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 x_ty) (ty_bits_u64 y_ty)))
+      (iadd y_ty (uextend y_ty x) y))
+(rule 3 (iadd_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 y_ty) (ty_bits_u64 x_ty)))
+      (iadd x_ty x (uextend x_ty y)))
+
+;; Emits an isub for two values. If they have different types
+;; then the smaller type is zero extended to the larger type.
+(decl isub_uextend (Value Value) Value)
+(rule 1 (isub_uextend x @ (value_type ty) y @ (value_type ty))
+      (isub ty x y))
+(rule 2 (isub_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 x_ty) (ty_bits_u64 y_ty)))
+      (isub y_ty (uextend y_ty x) y))
+(rule 3 (isub_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 y_ty) (ty_bits_u64 x_ty)))
+      (isub x_ty x (uextend x_ty y)))
+
+;; Try to group constants together so that other cprop rules can optimize them.
+;;
+;; (rotr (rotr x y) z) == (rotr x (iadd y z))
+;; (rotl (rotl x y) z) == (rotl x (iadd y z))
+;; (rotr (rotl x y) z) == (rotr x (isub y z))
+;; (rotl (rotr x y) z) == (rotl x (isub y z))
+;;
+;; if x or z are constants
+(rule (simplify (rotl ty (rotl ty x y @ (iconst _ _)) z)) (rotl ty x (iadd_uextend y z)))
+(rule (simplify (rotl ty (rotl ty x y) z @ (iconst _ _))) (rotl ty x (iadd_uextend y z)))
+(rule (simplify (rotr ty (rotr ty x y @ (iconst _ _)) z)) (rotr ty x (iadd_uextend y z)))
+(rule (simplify (rotr ty (rotr ty x y) z @ (iconst _ _))) (rotr ty x (iadd_uextend y z)))
+
+(rule (simplify (rotr ty (rotl ty x y @ (iconst _ _)) z)) (rotl ty x (isub_uextend y z)))
+(rule (simplify (rotr ty (rotl ty x y) z @ (iconst _ _))) (rotl ty x (isub_uextend y z)))
+(rule (simplify (rotl ty (rotr ty x y @ (iconst _ _)) z)) (rotr ty x (isub_uextend y z)))
+(rule (simplify (rotl ty (rotr ty x y) z @ (iconst _ _))) (rotr ty x (isub_uextend y z)))
+
+;; Similarly to the rules above, if y and z have the same type, we should emit
+;; an iadd or isub instead. In some backends this is cheaper than a rotate.
+;;
+;; If they have different types we end up in a situation where we have to insert
+;; and additional extend and that transformation is not universally beneficial.
+;;
+;; (rotr (rotr x y) z) == (rotr x (iadd y z))
+;; (rotl (rotl x y) z) == (rotl x (iadd y z))
+;; (rotr (rotl x y) z) == (rotl x (isub y z))
+;; (rotl (rotr x y) z) == (rotr x (isub y z))
+(rule (simplify (rotr ty (rotr ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotr ty x (iadd_uextend y z)))
+(rule (simplify (rotl ty (rotl ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotl ty x (iadd_uextend y z)))
+
+(rule (simplify (rotr ty (rotl ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotl ty x (isub_uextend y z)))
+(rule (simplify (rotl ty (rotr ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotr ty x (isub_uextend y z)))
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index eb03dd24a..c61876b8d 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -323,6 +323,10 @@
 (decl pure partial ty_half_width (Type) Type)
 (extern constructor ty_half_width ty_half_width)
 
+;; Generate a mask for the maximum shift amount for a given type. i.e 31 for I32.
+(decl pure ty_shift_mask (Type) u64)
+(rule (ty_shift_mask ty) (u64_sub (ty_bits (lane_type ty)) 1))
+
 ;; Compare two types for equality.
 (decl pure ty_equal (Type Type) bool)
 (extern constructor ty_equal ty_equal)
diff --git a/cranelift/codegen/src/prelude_opt.isle b/cranelift/codegen/src/prelude_opt.isle
index 099930f5b..921c7c65b 100644
--- a/cranelift/codegen/src/prelude_opt.isle
+++ b/cranelift/codegen/src/prelude_opt.isle
@@ -43,3 +43,16 @@
 ;; answer".
 (decl subsume (Value) Value)
 (extern constructor subsume subsume)
+
+;;;;; constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl iconst_u64 (Type u64) Value)
+
+;; Use a single iconst for types that fit in 64 bits.
+(rule 0 (iconst_u64 (ty_int_ref_scalar_64 ty) val)
+    (if-let $true (u64_le val (ty_umax ty)))
+    (iconst ty (imm64_masked ty val)))
+
+;; For i128 types use a iconst, but zero extend it to i128.
+(rule 1 (iconst_u64 $I128 val)
+    (uextend $I128 (iconst $I64 (imm64_masked $I64 val))))
diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h
index 44e4e3d2c..ce7d46e0f 100644
--- a/crates/c-api/include/wasmtime.h
+++ b/crates/c-api/include/wasmtime.h
@@ -181,6 +181,7 @@
 #define WASMTIME_API_H
 
 #include <wasi.h>
+// IWYU pragma: begin_exports
 #include <wasmtime/config.h>
 #include <wasmtime/engine.h>
 #include <wasmtime/error.h>
@@ -196,6 +197,7 @@
 #include <wasmtime/trap.h>
 #include <wasmtime/val.h>
 #include <wasmtime/async.h>
+// IWYU pragma: end_exports
 
 /**
  * \brief Wasmtime version string.
diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs
index 03900707a..d785435b6 100644
--- a/crates/cli-flags/src/lib.rs
+++ b/crates/cli-flags/src/lib.rs
@@ -65,6 +65,14 @@ wasmtime_option_group! {
         /// Enable the pooling allocator, in place of the on-demand allocator.
         pub pooling_allocator: Option<bool>,
 
+        /// How many bytes to keep resident between instantiations for the
+        /// pooling allocator in linear memories.
+        pub pooling_memory_keep_resident: Option<usize>,
+
+        /// How many bytes to keep resident between instantiations for the
+        /// pooling allocator in tables.
+        pub pooling_table_keep_resident: Option<usize>,
+
         /// Configure attempting to initialize linear memory via a
         /// copy-on-write mapping (default: yes)
         pub memory_init_cow: Option<bool>,
@@ -500,7 +508,14 @@ impl CommonOptions {
             ["pooling-allocator" : self.opts.pooling_allocator]
             enable => {
                 if enable {
-                    config.allocation_strategy(wasmtime::InstanceAllocationStrategy::pooling());
+                    let mut cfg = wasmtime::PoolingAllocationConfig::default();
+                    if let Some(size) = self.opts.pooling_memory_keep_resident {
+                        cfg.linear_memory_keep_resident(size);
+                    }
+                    if let Some(size) = self.opts.pooling_table_keep_resident {
+                        cfg.table_keep_resident(size);
+                    }
+                    config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(cfg));
                 }
             },
             true => err,
diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs
index 5b4909b48..517b5d1b9 100644
--- a/crates/jit/src/code_memory.rs
+++ b/crates/jit/src/code_memory.rs
@@ -149,22 +149,26 @@ impl CodeMemory {
     }
 
     /// Returns a reference to the underlying `MmapVec` this memory owns.
+    #[inline]
     pub fn mmap(&self) -> &MmapVec {
         &self.mmap
     }
 
     /// Returns the contents of the text section of the ELF executable this
     /// represents.
+    #[inline]
     pub fn text(&self) -> &[u8] {
         &self.mmap[self.text.clone()]
     }
 
     /// Returns the contents of the `ELF_WASMTIME_DWARF` section.
+    #[inline]
     pub fn dwarf(&self) -> &[u8] {
         &self.mmap[self.dwarf.clone()]
     }
 
     /// Returns the data in the `ELF_NAME_DATA` section.
+    #[inline]
     pub fn func_name_data(&self) -> &[u8] {
         &self.mmap[self.func_name_data.clone()]
     }
@@ -174,24 +178,28 @@ impl CodeMemory {
     ///
     /// This is used for initialization of memories and all data ranges stored
     /// in a `Module` are relative to the slice returned here.
+    #[inline]
     pub fn wasm_data(&self) -> &[u8] {
         &self.mmap[self.wasm_data.clone()]
     }
 
     /// Returns the encoded address map section used to pass to
     /// `wasmtime_environ::lookup_file_pos`.
+    #[inline]
     pub fn address_map_data(&self) -> &[u8] {
         &self.mmap[self.address_map_data.clone()]
     }
 
     /// Returns the contents of the `ELF_WASMTIME_INFO` section, or an empty
     /// slice if it wasn't found.
+    #[inline]
     pub fn wasmtime_info(&self) -> &[u8] {
         &self.mmap[self.info_data.clone()]
     }
 
     /// Returns the contents of the `ELF_WASMTIME_TRAPS` section, or an empty
     /// slice if it wasn't found.
+    #[inline]
     pub fn trap_data(&self) -> &[u8] {
         &self.mmap[self.trap_data.clone()]
     }
diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs
index 4f9f4c64d..341cd4dee 100644
--- a/crates/runtime/src/mmap.rs
+++ b/crates/runtime/src/mmap.rs
@@ -146,11 +146,13 @@ impl Mmap {
     }
 
     /// Return the allocated memory as a pointer to u8.
+    #[inline]
     pub fn as_ptr(&self) -> *const u8 {
         self.sys.as_ptr()
     }
 
     /// Return the allocated memory as a mutable pointer to u8.
+    #[inline]
     pub fn as_mut_ptr(&mut self) -> *mut u8 {
         self.sys.as_mut_ptr()
     }
@@ -159,6 +161,7 @@ impl Mmap {
     ///
     /// This is the byte length of this entire mapping which includes both
     /// addressible and non-addressible memory.
+    #[inline]
     pub fn len(&self) -> usize {
         self.sys.len()
     }
diff --git a/crates/runtime/src/mmap/unix.rs b/crates/runtime/src/mmap/unix.rs
index 3227ec3a0..97de914da 100644
--- a/crates/runtime/src/mmap/unix.rs
+++ b/crates/runtime/src/mmap/unix.rs
@@ -84,14 +84,17 @@ impl Mmap {
         Ok(())
     }
 
+    #[inline]
     pub fn as_ptr(&self) -> *const u8 {
         self.memory.as_ptr() as *const u8
     }
 
+    #[inline]
     pub fn as_mut_ptr(&mut self) -> *mut u8 {
         self.memory.as_ptr().cast()
     }
 
+    #[inline]
     pub fn len(&self) -> usize {
         unsafe { (*self.memory.as_ptr()).len() }
     }
diff --git a/crates/runtime/src/mmap/windows.rs b/crates/runtime/src/mmap/windows.rs
index 5ca208b33..4a8396c44 100644
--- a/crates/runtime/src/mmap/windows.rs
+++ b/crates/runtime/src/mmap/windows.rs
@@ -147,14 +147,17 @@ impl Mmap {
         Ok(())
     }
 
+    #[inline]
     pub fn as_ptr(&self) -> *const u8 {
         self.memory.as_ptr() as *const u8
     }
 
+    #[inline]
     pub fn as_mut_ptr(&mut self) -> *mut u8 {
         self.memory.as_ptr().cast()
     }
 
+    #[inline]
     pub fn len(&self) -> usize {
         unsafe { (*self.memory.as_ptr()).len() }
     }
diff --git a/crates/wast/Cargo.toml b/crates/wast/Cargo.toml
index 24b690427..b26451139 100644
--- a/crates/wast/Cargo.toml
+++ b/crates/wast/Cargo.toml
@@ -11,7 +11,7 @@ edition.workspace = true
 
 [dependencies]
 anyhow = { workspace = true }
-wasmtime = { workspace = true, features = ['cranelift'] }
+wasmtime = { workspace = true, features = ['cranelift', 'wat'] }
 wast = { workspace = true }
 log = { workspace = true }
 
diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs
index 98f7723c6..bd791784e 100644
--- a/winch/codegen/src/isa/x64/asm.rs
+++ b/winch/codegen/src/isa/x64/asm.rs
@@ -202,8 +202,14 @@ impl Assembler {
                 // instrunction.
                 let constant_data = pool.get(*c);
                 let data = VCodeConstantData::Pool(*c, constant_data.clone());
+                // If the constaant data is not marked as used, it will be
+                // inserted, therefore, it needs to be registered.
+                let needs_registration = !constants.pool_uses(&data);
                 let constant = constants.insert(VCodeConstantData::Pool(*c, constant_data.clone()));
-                buffer.register_constant(&constant, &data);
+
+                if needs_registration {
+                    buffer.register_constant(&constant, &data);
+                }
                 SyntheticAmode::ConstantOffset(constant)
             }
         }


Test patch:
diff --git a/cranelift/filetests/filetests/egraph/shifts.clif b/cranelift/filetests/filetests/egraph/shifts.clif
index 7852ada12..dbe984c84 100644
--- a/cranelift/filetests/filetests/egraph/shifts.clif
+++ b/cranelift/filetests/filetests/egraph/shifts.clif
@@ -524,3 +524,237 @@ block0(v0: i8, v1: i16, v2: i16):
 
 ; check: v5 = rotl v0, v1
 ; check: return v5
+
+function %ishl_prop_overflow(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 7
+    v2 = iconst.i8 7
+    v3 = ishl.i8 v0, v1
+    v4 = ishl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 0
+; check: return v5  ; v5 = 0
+
+function %ishl_prop_type_diff(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i16 7
+    v2 = iconst.i16 7
+    v3 = ishl.i8 v0, v1
+    v4 = ishl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 0
+; check: return v5  ; v5 = 0
+
+function %ushr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = ushr.i8 v0, v1
+    v4 = ushr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 2
+; check: v6 = ushr v0, v5  ; v5 = 2
+; check: return v6
+
+function %ushr_prop_overflow(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 7
+    v2 = iconst.i8 7
+    v3 = ushr.i8 v0, v1
+    v4 = ushr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 0
+; check: return v5  ; v5 = 0
+
+
+function %sshr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = sshr.i8 v0, v1
+    v4 = sshr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 2
+; check: v6 = sshr v0, v5  ; v5 = 2
+; check: return v6
+
+function %i128_ushr_overflow_becomes_0(i128) -> i128 {
+block0(v0: i128):
+    v1 = iconst.i16 200
+    v2 = ushr v0, v1
+    v3 = ushr v2, v1
+    return v3
+}
+
+; check: v4 = iconst.i64 0
+; nextln: v5 = uextend.i128 v4  ; v4 = 0
+; nextln: return v5
+
+
+function %i128_ishl_overflow_becomes_0(i128) -> i128 {
+block0(v0: i128):
+    v1 = iconst.i16 200
+    v2 = ishl v0, v1
+    v3 = ishl v2, v1
+    return v3
+}
+
+; check: v4 = iconst.i64 0
+; nextln: v5 = uextend.i128 v4  ; v4 = 0
+; nextln: return v5
+
+function %simd_shift_does_not_panic(i8x16) -> i8x16 {
+block0(v0: i8x16):
+    v1 = iconst.i64 0
+    v2 = ushr v0, v1
+    v3 = ushr v2, v1
+    return v3
+}
+
+; check: v1 = iconst.i64 0
+; nextln: v2 = ushr v0, v1  ; v1 = 0
+; check: return v2
+
+function %merges_shift_amount_based_on_lane_type(i8x16) -> i8x16 {
+block0(v0: i8x16):
+    v1 = iconst.i16 30
+    v2 = iconst.i16 3
+
+    v3 = sshr v0, v1
+    v4 = sshr v3, v2
+    v5 = sshr v4, v2
+    return v5
+}
+
+; check: v1 = iconst.i16 30
+; nextln: v3 = sshr v0, v1  ; v1 = 30
+; nextln: v6 = iconst.i16 6
+; nextln: v7 = sshr v3, v6  ; v6 = 6
+; check: return v7
+
+
+function %rotr_rotr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = rotr.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v11 = iconst.i8 2
+; check: v13 = rotr v0, v11  ; v11 = 2
+; check: return v13
+
+function %rotr_rotr_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotr.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iadd v1, v2
+; check: v6 = rotr v0, v5
+; check: return v6
+
+function %rotl_rotl_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = rotl.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v11 = iconst.i8 2
+; check: v13 = rotl v0, v11  ; v11 = 2
+; check: return v13
+
+function %rotl_rotl_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotl.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iadd v1, v2
+; check: v6 = rotl v0, v5
+; check: return v6
+
+
+function %rotl_rotr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 2
+    v2 = iconst.i8 1
+    v3 = rotr.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v2 = iconst.i8 1
+; check: v21 = rotr v0, v2  ; v2 = 1
+; check: return v21
+
+function %rotl_rotr_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotr.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = isub v1, v2
+; check: v6 = rotr v0, v5
+; check: return v6
+
+function %rotl_rotr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 2
+    v2 = iconst.i8 1
+    v3 = rotl.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v2 = iconst.i8 1
+; check: v21 = rotl v0, v2  ; v2 = 1
+; check: return v21
+
+function %rotl_rotr_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotl.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = isub v1, v2
+; check: v6 = rotl v0, v5
+; check: return v6
+
+function %rotl_rotr_subsume(i8, i8) -> i8 {
+block0(v0: i8, v1: i8):
+    v2 = rotl.i8 v0, v1
+    v3 = rotr.i8 v2, v1
+    return v3
+}
+
+; check: return v0
+
+
+function %rotr_rotl_subsume(i8, i8) -> i8 {
+block0(v0: i8, v1: i8):
+    v2 = rotr.i8 v0, v1
+    v3 = rotl.i8 v2, v1
+    return v3
+}
+
+; check: return v0
diff --git a/cranelift/filetests/filetests/isa/aarch64/condbr.clif b/cranelift/filetests/filetests/isa/aarch64/condbr.clif
index f0661556d..13d59de14 100644
--- a/cranelift/filetests/filetests/isa/aarch64/condbr.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/condbr.clif
@@ -733,3 +733,183 @@ block1:
 ; block1: ; offset 0x1c
 ;   ret
 
+function %tbnz_i8(i8) {
+block0(v0: i8):
+  v1 = band_imm v0, 0x10
+  brif v1, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbnz x0, #4, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbnz w0, #4, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+function %tbz_i16(i16) {
+block0(v0: i16):
+  v1 = band_imm v0, 0x1000
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbz x0, #12, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbz w0, #0xc, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+function %tbnz_i32(i32) {
+block0(v0: i32):
+  v1 = band_imm v0, 0x10000
+  brif v1, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbnz x0, #16, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbnz w0, #0x10, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+
+function %tbz_i64(i64) {
+block0(v0: i64):
+  v1 = band_imm v0, 0x1_00000000
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbz x0, #32, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbz x0, #0x20, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+function %not_tbz1(i8) {
+block0(v0: i8):
+  v1 = band_imm v0, 0x100
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   movz w4, #0
+;   and w4, w0, w4
+;   uxtb w4, w4
+;   subs wzr, w4, #0
+;   b.eq label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   mov w4, #0
+;   and w4, w0, w4
+;   uxtb w4, w4
+;   cmp w4, #0
+;   b.eq #0x18
+; block1: ; offset 0x14
+;   ret
+; block2: ; offset 0x18
+;   ret
+
+function %not_tbz2(i8) {
+block0(v0: i8):
+  v1 = band_imm v0, 0x3
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   and w3, w0, #3
+;   uxtb w3, w3
+;   subs wzr, w3, #0
+;   b.eq label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   and w3, w0, #3
+;   uxtb w3, w3
+;   cmp w3, #0
+;   b.eq #0x14
+; block1: ; offset 0x10
+;   ret
+; block2: ; offset 0x14
+;   ret
diff --git a/tests/misc_testsuite/winch/local_get.wast b/tests/misc_testsuite/winch/local_get.wast
new file mode 100644
index 000000000..16e06113d
--- /dev/null
+++ b/tests/misc_testsuite/winch/local_get.wast
@@ -0,0 +1,103 @@
+;; Test `local.get` operator
+
+(module
+  ;; Typing
+
+  (func (export "type-local-i32") (result i32) (local i32) (local.get 0))
+  (func (export "type-local-i64") (result i64) (local i64) (local.get 0))
+  (func (export "type-local-f32") (result f32) (local f32) (local.get 0))
+  (func (export "type-local-f64") (result f64) (local f64) (local.get 0))
+
+  (func (export "type-param-i32") (param i32) (result i32) (local.get 0))
+  (func (export "type-param-i64") (param i64) (result i64) (local.get 0))
+  (func (export "type-param-f32") (param f32) (result f32) (local.get 0))
+  (func (export "type-param-f64") (param f64) (result f64) (local.get 0))
+
+  (func (export "type-mixed") (param i64 f32 f64 i32 i32)
+    (local f32 i64 i64 f64)
+    (drop (i64.eqz (local.get 0)))
+    (drop (f32.neg (local.get 1)))
+    (drop (f64.neg (local.get 2)))
+    (drop (i32.eqz (local.get 3)))
+    (drop (i32.eqz (local.get 4)))
+    (drop (f32.neg (local.get 5)))
+    (drop (i64.eqz (local.get 6)))
+    (drop (i64.eqz (local.get 7)))
+    (drop (f64.neg (local.get 8)))
+  )
+
+
+  ;; As parameter of control constructs and instructions
+
+  (func (export "as-block-value") (param i32) (result i32)
+    (block (result i32) (local.get 0))
+  )
+  (func (export "as-loop-value") (param i32) (result i32)
+    (loop (result i32) (local.get 0))
+  )
+  (func (export "as-br-value") (param i32) (result i32)
+    (block (result i32) (br 0 (local.get 0)))
+  )
+  (func (export "as-br_if-value") (param i32) (result i32)
+    (block $l0 (result i32) (br_if $l0 (local.get 0) (i32.const 1)))
+  )
+
+  (func (export "as-br_if-value-cond") (param i32) (result i32)
+    (block (result i32)
+      (br_if 0 (local.get 0) (local.get 0))
+    )
+  )
+  (func (export "as-br_table-value") (param i32) (result i32)
+    (block
+      (block
+        (block
+          (br_table 0 1 2 (local.get 0))
+          (return (i32.const 0))
+        )
+        (return (i32.const 1))
+      )
+      (return (i32.const 2))
+    )
+    (i32.const 3)
+  )
+
+  (func (export "as-return-value") (param i32) (result i32)
+    (return (local.get 0))
+  )
+
+  (func (export "as-if-then") (param i32) (result i32)
+    (if (result i32) (local.get 0) (then (local.get 0)) (else (i32.const 0)))
+  )
+  (func (export "as-if-else") (param i32) (result i32)
+    (if (result i32) (local.get 0) (then (i32.const 1)) (else (local.get 0)))
+  )
+)
+
+(assert_return (invoke "type-local-i32") (i32.const 0))
+(assert_return (invoke "type-local-i64") (i64.const 0))
+(assert_return (invoke "type-local-f32") (f32.const 0))
+(assert_return (invoke "type-local-f64") (f64.const 0))
+
+(assert_return (invoke "type-param-i32" (i32.const 2)) (i32.const 2))
+(assert_return (invoke "type-param-i64" (i64.const 3)) (i64.const 3))
+(assert_return (invoke "type-param-f32" (f32.const 4.4)) (f32.const 4.4))
+(assert_return (invoke "type-param-f64" (f64.const 5.5)) (f64.const 5.5))
+
+(assert_return (invoke "as-block-value" (i32.const 6)) (i32.const 6))
+(assert_return (invoke "as-loop-value" (i32.const 7)) (i32.const 7))
+
+(assert_return (invoke "as-br-value" (i32.const 8)) (i32.const 8))
+(assert_return (invoke "as-br_if-value" (i32.const 9)) (i32.const 9))
+(assert_return (invoke "as-br_if-value-cond" (i32.const 10)) (i32.const 10))
+(assert_return (invoke "as-br_table-value" (i32.const 1)) (i32.const 2))
+
+(assert_return (invoke "as-return-value" (i32.const 0)) (i32.const 0))
+
+(assert_return (invoke "as-if-then" (i32.const 1)) (i32.const 1))
+(assert_return (invoke "as-if-else" (i32.const 0)) (i32.const 0))
+
+(assert_return
+  (invoke "type-mixed"
+    (i64.const 1) (f32.const 2.2) (f64.const 3.3) (i32.const 4) (i32.const 5)
+  )
+)
diff --git a/tests/misc_testsuite/winch/local_set.wast b/tests/misc_testsuite/winch/local_set.wast
new file mode 100644
index 000000000..f368f7bf3
--- /dev/null
+++ b/tests/misc_testsuite/winch/local_set.wast
@@ -0,0 +1,97 @@
+;; Test `local.set` operator
+
+(module
+  ;; Typing
+
+  (func (export "type-local-i32") (local i32) (local.set 0 (i32.const 0)))
+  (func (export "type-local-i64") (local i64) (local.set 0 (i64.const 0)))
+  (func (export "type-local-f32") (local f32) (local.set 0 (f32.const 0)))
+  (func (export "type-local-f64") (local f64) (local.set 0 (f64.const 0)))
+
+  (func (export "type-param-i32") (param i32) (local.set 0 (i32.const 10)))
+  (func (export "type-param-i64") (param i64) (local.set 0 (i64.const 11)))
+  (func (export "type-param-f32") (param f32) (local.set 0 (f32.const 11.1)))
+  (func (export "type-param-f64") (param f64) (local.set 0 (f64.const 12.2)))
+
+  (func (export "type-mixed") (param i64 f32 f64 i32 i32) (local f32 i64 i64 f64)
+    (local.set 0 (i64.const 0))
+    (local.set 1 (f32.const 0))
+    (local.set 2 (f64.const 0))
+    (local.set 3 (i32.const 0))
+    (local.set 4 (i32.const 0))
+    (local.set 5 (f32.const 0))
+    (local.set 6 (i64.const 0))
+    (local.set 7 (i64.const 0))
+    (local.set 8 (f64.const 0))
+  )
+
+  ;; As parameter of control constructs and instructions
+
+  (func (export "as-block-value") (param i32)
+    (block (local.set 0 (i32.const 1)))
+  )
+  (func (export "as-loop-value") (param i32)
+    (loop (local.set 0 (i32.const 3)))
+  )
+
+  (func (export "as-br-value") (param i32)
+    (block (br 0 (local.set 0 (i32.const 9))))
+  )
+  (func (export "as-br_if-value") (param i32)
+    (block
+      (br_if 0 (local.set 0 (i32.const 8)) (i32.const 1))
+    )
+  )
+  (func (export "as-br_if-value-cond") (param i32)
+    (block
+      (br_if 0 (i32.const 6) (local.set 0 (i32.const 9)))
+    )
+  )
+  (func (export "as-br_table-value") (param i32)
+    (block
+      (br_table 0 (local.set 0 (i32.const 10)) (i32.const 1))
+    )
+  )
+
+  (func (export "as-return-value") (param i32)
+    (return (local.set 0 (i32.const 7)))
+  )
+
+  (func (export "as-if-then") (param i32)
+    (if (local.get 0) (then (local.set 0 (i32.const 3))))
+  )
+  (func (export "as-if-else") (param i32)
+    (if (local.get 0) (then) (else (local.set 0 (i32.const 1))))
+  )
+)
+
+(assert_return (invoke "type-local-i32"))
+(assert_return (invoke "type-local-i64"))
+(assert_return (invoke "type-local-f32"))
+(assert_return (invoke "type-local-f64"))
+
+(assert_return (invoke "type-param-i32" (i32.const 2)))
+(assert_return (invoke "type-param-i64" (i64.const 3)))
+(assert_return (invoke "type-param-f32" (f32.const 4.4)))
+(assert_return (invoke "type-param-f64" (f64.const 5.5)))
+
+(assert_return (invoke "as-block-value" (i32.const 0)))
+(assert_return (invoke "as-loop-value" (i32.const 0)))
+
+(assert_return (invoke "as-br-value" (i32.const 0)))
+(assert_return (invoke "as-br_if-value" (i32.const 0)))
+(assert_return (invoke "as-br_if-value-cond" (i32.const 0)))
+(assert_return (invoke "as-br_table-value" (i32.const 0)))
+
+(assert_return (invoke "as-return-value" (i32.const 0)))
+
+(assert_return (invoke "as-if-then" (i32.const 1)))
+(assert_return (invoke "as-if-else" (i32.const 0)))
+
+(assert_return
+  (invoke "type-mixed"
+    (i64.const 1) (f32.const 2.2) (f64.const 3.3) (i32.const 4) (i32.const 5)
+  )
+)
+
+