Repository: bytecodealliance/wasmtime
Pull request #7455 (2023-11-02 16:37:26)

Print the error from the wasi-http incoming-handler to stderr
Prior to this change, the error was never printed out, making it a challenge to figure out what caused the error




Patch:
diff --git a/Cargo.toml b/Cargo.toml
index b9f539873..513105811 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -343,7 +343,7 @@ component-model = [
   "wasmtime-wast/component-model",
   "wasmtime-cli-flags/component-model"
 ]
-wat = ["dep:wat"]
+wat = ["dep:wat", "wasmtime/wat"]
 cache = ["dep:wasmtime-cache", "wasmtime-cli-flags/cache"]
 parallel-compilation = ["wasmtime-cli-flags/parallel-compilation"]
 logging = ["wasmtime-cli-flags/logging"]
diff --git a/cranelift/codegen/src/isa/aarch64/inst.isle b/cranelift/codegen/src/isa/aarch64/inst.isle
index 90c091118..314d09b15 100644
--- a/cranelift/codegen/src/isa/aarch64/inst.isle
+++ b/cranelift/codegen/src/isa/aarch64/inst.isle
@@ -833,6 +833,15 @@
         (not_taken BranchTarget)
         (kind CondBrKind))
 
+       ;; A conditional branch which tests the `bit` of `rn` and branches
+       ;; depending on `kind`.
+       (TestBitAndBranch
+        (kind TestBitAndBranchKind)
+        (taken BranchTarget)
+        (not_taken BranchTarget)
+        (rn Reg)
+        (bit u8))
+
        ;; A conditional trap: execute a `udf` if the condition is true. This is
        ;; one VCode instruction because it uses embedded control flow; it is
        ;; logically a single-in, single-out region, but needs to appear as one
@@ -1203,6 +1212,8 @@
       (enum Size32
             Size64))
 
+(type TestBitAndBranchKind (enum (Z) (NZ)))
+
 ;; Helper for calculating the `OperandSize` corresponding to a type
 (decl operand_size (Type) OperandSize)
 (rule 1 (operand_size (fits_in_32 _ty)) (OperandSize.Size32))
@@ -4079,6 +4090,21 @@
       (ConsumesFlags.ConsumesFlagsSideEffect
        (MInst.CondBr taken not_taken kind)))
 
+;; Helper for emitting `MInst.TestBitAndBranch` instructions.
+(decl test_branch (TestBitAndBranchKind BranchTarget BranchTarget Reg u8) SideEffectNoResult)
+(rule (test_branch kind taken not_taken rn bit)
+      (SideEffectNoResult.Inst (MInst.TestBitAndBranch kind taken not_taken rn bit)))
+
+;; Helper for emitting `tbnz` instructions.
+(decl tbnz (BranchTarget BranchTarget Reg u8) SideEffectNoResult)
+(rule (tbnz taken not_taken rn bit)
+      (test_branch (TestBitAndBranchKind.NZ) taken not_taken rn bit))
+
+;; Helper for emitting `tbz` instructions.
+(decl tbz (BranchTarget BranchTarget Reg u8) SideEffectNoResult)
+(rule (tbz taken not_taken rn bit)
+      (test_branch (TestBitAndBranchKind.Z) taken not_taken rn bit))
+
 ;; Helper for emitting `MInst.MovToNZCV` instructions.
 (decl mov_to_nzcv (Reg) ProducesFlags)
 (rule (mov_to_nzcv rn)
diff --git a/cranelift/codegen/src/isa/aarch64/inst/args.rs b/cranelift/codegen/src/isa/aarch64/inst/args.rs
index c6394e659..f6d7f5800 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/args.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/args.rs
@@ -340,26 +340,31 @@ impl BranchTarget {
         }
     }
 
+    /// Return the target's offset, if specified, or zero if label-based.
+    pub fn as_offset14_or_zero(self) -> u32 {
+        self.as_offset_bounded(14)
+    }
+
     /// Return the target's offset, if specified, or zero if label-based.
     pub fn as_offset19_or_zero(self) -> u32 {
-        let off = match self {
-            BranchTarget::ResolvedOffset(off) => off >> 2,
-            _ => 0,
-        };
-        assert!(off <= 0x3ffff);
-        assert!(off >= -0x40000);
-        (off as u32) & 0x7ffff
+        self.as_offset_bounded(19)
     }
 
     /// Return the target's offset, if specified, or zero if label-based.
     pub fn as_offset26_or_zero(self) -> u32 {
+        self.as_offset_bounded(26)
+    }
+
+    fn as_offset_bounded(self, bits: u32) -> u32 {
         let off = match self {
             BranchTarget::ResolvedOffset(off) => off >> 2,
             _ => 0,
         };
-        assert!(off <= 0x1ffffff);
-        assert!(off >= -0x2000000);
-        (off as u32) & 0x3ffffff
+        let hi = (1 << (bits - 1)) - 1;
+        let lo = -(1 << bits - 1);
+        assert!(off <= hi);
+        assert!(off >= lo);
+        (off as u32) & ((1 << bits) - 1)
     }
 }
 
@@ -764,3 +769,15 @@ impl APIKey {
         0xd503201f | (crm << 8) | (op2 << 5)
     }
 }
+
+pub use crate::isa::aarch64::lower::isle::generated_code::TestBitAndBranchKind;
+
+impl TestBitAndBranchKind {
+    /// Complements this branch condition to act on the opposite result.
+    pub fn complement(&self) -> TestBitAndBranchKind {
+        match self {
+            TestBitAndBranchKind::Z => TestBitAndBranchKind::NZ,
+            TestBitAndBranchKind::NZ => TestBitAndBranchKind::Z,
+        }
+    }
+}
diff --git a/cranelift/codegen/src/isa/aarch64/inst/emit.rs b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
index 39a49dfdd..7549ee4a6 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/emit.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/emit.rs
@@ -172,6 +172,27 @@ fn enc_conditional_br(
     }
 }
 
+fn enc_test_bit_and_branch(
+    kind: TestBitAndBranchKind,
+    taken: BranchTarget,
+    reg: Reg,
+    bit: u8,
+) -> u32 {
+    assert!(bit < 64);
+    let op_31 = u32::from(bit >> 5);
+    let op_23_19 = u32::from(bit & 0b11111);
+    let op_30_24 = 0b0110110
+        | match kind {
+            TestBitAndBranchKind::Z => 0,
+            TestBitAndBranchKind::NZ => 1,
+        };
+    (op_31 << 31)
+        | (op_30_24 << 24)
+        | (op_23_19 << 19)
+        | (taken.as_offset14_or_zero() << 5)
+        | machreg_to_gpr(reg)
+}
+
 fn enc_move_wide(op: MoveWideOp, rd: Writable<Reg>, imm: MoveWideConst, size: OperandSize) -> u32 {
     assert!(imm.shift <= 0b11);
     let op = match op {
@@ -3224,6 +3245,32 @@ impl MachInstEmit for Inst {
                 }
                 sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
             }
+            &Inst::TestBitAndBranch {
+                taken,
+                not_taken,
+                kind,
+                rn,
+                bit,
+            } => {
+                let rn = allocs.next(rn);
+                // Emit the conditional branch first
+                let cond_off = sink.cur_offset();
+                if let Some(l) = taken.as_label() {
+                    sink.use_label_at_offset(cond_off, l, LabelUse::Branch14);
+                    let inverted =
+                        enc_test_bit_and_branch(kind.complement(), taken, rn, bit).to_le_bytes();
+                    sink.add_cond_branch(cond_off, cond_off + 4, l, &inverted[..]);
+                }
+                sink.put4(enc_test_bit_and_branch(kind, taken, rn, bit));
+
+                // Unconditional part next.
+                let uncond_off = sink.cur_offset();
+                if let Some(l) = not_taken.as_label() {
+                    sink.use_label_at_offset(uncond_off, l, LabelUse::Branch26);
+                    sink.add_uncond_branch(uncond_off, uncond_off + 4, l);
+                }
+                sink.put4(enc_jump26(0b000101, not_taken.as_offset26_or_zero()));
+            }
             &Inst::TrapIf { kind, trap_code } => {
                 let label = sink.defer_trap(trap_code, state.take_stack_map());
                 // condbr KIND, LABEL
diff --git a/cranelift/codegen/src/isa/aarch64/inst/mod.rs b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
index bf3bd50e6..d6027c074 100644
--- a/cranelift/codegen/src/isa/aarch64/inst/mod.rs
+++ b/cranelift/codegen/src/isa/aarch64/inst/mod.rs
@@ -902,6 +902,9 @@ fn aarch64_get_operands<F: Fn(VReg) -> VReg>(inst: &Inst, collector: &mut Operan
             }
             CondBrKind::Cond(_) => {}
         },
+        &Inst::TestBitAndBranch { rn, .. } => {
+            collector.reg_use(rn);
+        }
         &Inst::IndirectBr { rn, .. } => {
             collector.reg_use(rn);
         }
@@ -1043,6 +1046,7 @@ impl MachInst for Inst {
             &Inst::ReturnCall { .. } | &Inst::ReturnCallInd { .. } => MachTerminator::RetCall,
             &Inst::Jump { .. } => MachTerminator::Uncond,
             &Inst::CondBr { .. } => MachTerminator::Cond,
+            &Inst::TestBitAndBranch { .. } => MachTerminator::Cond,
             &Inst::IndirectBr { .. } => MachTerminator::Indirect,
             &Inst::JTSequence { .. } => MachTerminator::Indirect,
             _ => MachTerminator::None,
@@ -2663,6 +2667,22 @@ impl Inst {
                     }
                 }
             }
+            &Inst::TestBitAndBranch {
+                kind,
+                ref taken,
+                ref not_taken,
+                rn,
+                bit,
+            } => {
+                let cond = match kind {
+                    TestBitAndBranchKind::Z => "z",
+                    TestBitAndBranchKind::NZ => "nz",
+                };
+                let taken = taken.pretty_print(0, allocs);
+                let not_taken = not_taken.pretty_print(0, allocs);
+                let rn = pretty_print_reg(rn, allocs);
+                format!("tb{cond} {rn}, #{bit}, {taken} ; b {not_taken}")
+            }
             &Inst::IndirectBr { rn, .. } => {
                 let rn = pretty_print_reg(rn, allocs);
                 format!("br {}", rn)
@@ -2882,6 +2902,9 @@ impl Inst {
 /// Different forms of label references for different instruction formats.
 #[derive(Clone, Copy, Debug, PartialEq, Eq)]
 pub enum LabelUse {
+    /// 14-bit branch offset (conditional branches). PC-rel, offset is imm <<
+    /// 2. Immediate is 14 signed bits, in bits 18:5. Used by tbz and tbnz.
+    Branch14,
     /// 19-bit branch offset (conditional branches). PC-rel, offset is imm << 2. Immediate is 19
     /// signed bits, in bits 23:5. Used by cbz, cbnz, b.cond.
     Branch19,
@@ -2908,8 +2931,10 @@ impl MachInstLabelUse for LabelUse {
     /// Maximum PC-relative range (positive), inclusive.
     fn max_pos_range(self) -> CodeOffset {
         match self {
-            // 19-bit immediate, left-shifted by 2, for 21 bits of total range. Signed, so +2^20
-            // from zero. Likewise for two other shifted cases below.
+            // N-bit immediate, left-shifted by 2, for (N+2) bits of total
+            // range. Signed, so +2^(N+1) from zero. Likewise for two other
+            // shifted cases below.
+            LabelUse::Branch14 => (1 << 15) - 1,
             LabelUse::Branch19 => (1 << 20) - 1,
             LabelUse::Branch26 => (1 << 27) - 1,
             LabelUse::Ldr19 => (1 << 20) - 1,
@@ -2941,6 +2966,7 @@ impl MachInstLabelUse for LabelUse {
         let pc_rel = pc_rel as u32;
         let insn_word = u32::from_le_bytes([buffer[0], buffer[1], buffer[2], buffer[3]]);
         let mask = match self {
+            LabelUse::Branch14 => 0x0007ffe0, // bits 18..5 inclusive
             LabelUse::Branch19 => 0x00ffffe0, // bits 23..5 inclusive
             LabelUse::Branch26 => 0x03ffffff, // bits 25..0 inclusive
             LabelUse::Ldr19 => 0x00ffffe0,    // bits 23..5 inclusive
@@ -2955,6 +2981,7 @@ impl MachInstLabelUse for LabelUse {
             }
         };
         let pc_rel_inserted = match self {
+            LabelUse::Branch14 => (pc_rel_shifted & 0x3fff) << 5,
             LabelUse::Branch19 | LabelUse::Ldr19 => (pc_rel_shifted & 0x7ffff) << 5,
             LabelUse::Branch26 => pc_rel_shifted & 0x3ffffff,
             LabelUse::Adr21 => (pc_rel_shifted & 0x7ffff) << 5 | (pc_rel_shifted & 0x180000) << 10,
@@ -2975,8 +3002,8 @@ impl MachInstLabelUse for LabelUse {
     /// Is a veneer supported for this label reference type?
     fn supports_veneer(self) -> bool {
         match self {
-            LabelUse::Branch19 => true, // veneer is a Branch26
-            LabelUse::Branch26 => true, // veneer is a PCRel32
+            LabelUse::Branch14 | LabelUse::Branch19 => true, // veneer is a Branch26
+            LabelUse::Branch26 => true,                      // veneer is a PCRel32
             _ => false,
         }
     }
@@ -2984,7 +3011,7 @@ impl MachInstLabelUse for LabelUse {
     /// How large is the veneer, if supported?
     fn veneer_size(self) -> CodeOffset {
         match self {
-            LabelUse::Branch19 => 4,
+            LabelUse::Branch14 | LabelUse::Branch19 => 4,
             LabelUse::Branch26 => 20,
             _ => unreachable!(),
         }
@@ -3002,7 +3029,7 @@ impl MachInstLabelUse for LabelUse {
         veneer_offset: CodeOffset,
     ) -> (CodeOffset, LabelUse) {
         match self {
-            LabelUse::Branch19 => {
+            LabelUse::Branch14 | LabelUse::Branch19 => {
                 // veneer is a Branch26 (unconditional branch). Just encode directly here -- don't
                 // bother with constructing an Inst.
                 let insn_word = 0b000101 << 26;
diff --git a/cranelift/codegen/src/isa/aarch64/lower.isle b/cranelift/codegen/src/isa/aarch64/lower.isle
index 4d8b29fb6..7af210286 100644
--- a/cranelift/codegen/src/isa/aarch64/lower.isle
+++ b/cranelift/codegen/src/isa/aarch64/lower.isle
@@ -2882,6 +2882,24 @@
         (with_flags_side_effect flags
          (cond_br taken not_taken (cond_br_not_zero rt))))))
 
+;; Special lowerings for `tbnz` - "Test bit and Branch if Nonzero"
+(rule 1 (lower_branch (brif (band x @ (value_type ty) (u64_from_iconst n)) _ _)
+                     (two_targets taken not_taken))
+  (if-let bit (test_and_compare_bit_const ty n))
+  (emit_side_effect (tbnz taken not_taken x bit)))
+
+;; Special lowering for `tbz` - "Test bit and Branch if Zero"
+(rule 1 (lower_branch (brif (icmp (IntCC.Equal)
+                                  (band x @ (value_type (fits_in_64 ty))
+                                        (u64_from_iconst n))
+                                  (u64_from_iconst 0)) _ _)
+                     (two_targets taken not_taken))
+  (if-let bit (test_and_compare_bit_const ty n))
+  (emit_side_effect (tbz taken not_taken x bit)))
+
+(decl pure partial test_and_compare_bit_const (Type u64) u8)
+(extern constructor test_and_compare_bit_const test_and_compare_bit_const)
+
 ;;; Rules for `jump` ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
 
 (rule (lower_branch (jump _) (single_target label))
diff --git a/cranelift/codegen/src/isa/aarch64/lower/isle.rs b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
index 0c3f151c4..1208f519c 100644
--- a/cranelift/codegen/src/isa/aarch64/lower/isle.rs
+++ b/cranelift/codegen/src/isa/aarch64/lower/isle.rs
@@ -812,4 +812,15 @@ impl Context for IsleContext<'_, '_, MInst, AArch64Backend> {
     fn uimm12_scaled_from_i64(&mut self, val: i64, ty: Type) -> Option<UImm12Scaled> {
         UImm12Scaled::maybe_from_i64(val, ty)
     }
+
+    fn test_and_compare_bit_const(&mut self, ty: Type, n: u64) -> Option<u8> {
+        if n.count_ones() != 1 {
+            return None;
+        }
+        let bit = n.trailing_zeros();
+        if bit >= ty.bits() {
+            return None;
+        }
+        Some(bit as u8)
+    }
 }
diff --git a/cranelift/codegen/src/isa/riscv64/inst.isle b/cranelift/codegen/src/isa/riscv64/inst.isle
index 8627f2a29..0c4e1e3b5 100644
--- a/cranelift/codegen/src/isa/riscv64/inst.isle
+++ b/cranelift/codegen/src/isa/riscv64/inst.isle
@@ -1783,9 +1783,6 @@
 
 
 
-;; Generate a mask for the bit-width of the given type
-(decl pure shift_mask (Type) u64)
-(rule (shift_mask ty) (u64_sub (ty_bits (lane_type ty)) 1))
 
 ;; Helper for generating a i64 from a pair of Imm20 and Imm12 constants
 (decl i64_generate_imm (Imm20 Imm12) i64)
diff --git a/cranelift/codegen/src/isa/riscv64/lower.isle b/cranelift/codegen/src/isa/riscv64/lower.isle
index 0d3a271f4..e3fa6a9f2 100644
--- a/cranelift/codegen/src/isa/riscv64/lower.isle
+++ b/cranelift/codegen/src/isa/riscv64/lower.isle
@@ -1193,7 +1193,7 @@
 
 ;; 8/16 bit types need a mask on the shift amount
 (rule 0 (lower (has_type (ty_int (ty_8_or_16 ty)) (ishl x y)))
-  (if-let mask (u64_to_imm12 (shift_mask ty)))
+  (if-let mask (u64_to_imm12 (ty_shift_mask ty)))
   (rv_sllw x (rv_andi (value_regs_get y 0) mask)))
 
 ;; Using the 32bit version of `sll` automatically masks the shift amount.
@@ -1206,12 +1206,12 @@
 
 ;; If the shift amount is known. We can mask it and encode it in the instruction.
 (rule 2 (lower (has_type (int_fits_in_32 ty) (ishl x (maybe_uextend (imm12_from_value y)))))
-  (rv_slliw x (imm12_and y (shift_mask ty))))
+  (rv_slliw x (imm12_and y (ty_shift_mask ty))))
 
 ;; We technically don't need to mask the shift amount here. The instruction
 ;; does the right thing. But it's neater when pretty printing it.
 (rule 3 (lower (has_type ty @ $I64 (ishl x (maybe_uextend (imm12_from_value y)))))
-  (rv_slli x (imm12_and y (shift_mask ty))))
+  (rv_slli x (imm12_and y (ty_shift_mask ty))))
 
 ;; With `Zba` we have a shift that zero extends the LHS argument.
 (rule 4 (lower (has_type $I64 (ishl (uextend x @ (value_type $I32)) (maybe_uextend (imm12_from_value y)))))
@@ -1253,7 +1253,7 @@
 ;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be
 ;; zero extended.
 (rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x y)))
-  (if-let mask (u64_to_imm12 (shift_mask ty)))
+  (if-let mask (u64_to_imm12 (ty_shift_mask ty)))
   (rv_srlw (zext x) (rv_andi (value_regs_get y 0) mask)))
 
 ;; Using the 32bit version of `srl` automatically masks the shift amount.
@@ -1266,7 +1266,7 @@
 
 ;; When the RHS is known we can just encode it in the instruction.
 (rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (ushr x (maybe_uextend (imm12_from_value y)))))
-  (rv_srliw (zext x) (imm12_and y (shift_mask ty))))
+  (rv_srliw (zext x) (imm12_and y (ty_shift_mask ty))))
 
 (rule 3 (lower (has_type $I32 (ushr x (maybe_uextend (imm12_from_value y)))))
   (rv_srliw x y))
@@ -1308,7 +1308,7 @@
 ;; 8/16 bit types need a mask on the shift amount, and the LHS needs to be
 ;; zero extended.
 (rule 0 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x y)))
-  (if-let mask (u64_to_imm12 (shift_mask ty)))
+  (if-let mask (u64_to_imm12 (ty_shift_mask ty)))
   (rv_sraw (sext x) (rv_andi (value_regs_get y 0) mask)))
 
 ;; Using the 32bit version of `sra` automatically masks the shift amount.
@@ -1321,7 +1321,7 @@
 
 ;; When the RHS is known we can just encode it in the instruction.
 (rule 2 (lower (has_type (ty_int (fits_in_16 ty)) (sshr x (maybe_uextend (imm12_from_value y)))))
-  (rv_sraiw (sext x) (imm12_and y (shift_mask ty))))
+  (rv_sraiw (sext x) (imm12_and y (ty_shift_mask ty))))
 
 (rule 3 (lower (has_type $I32 (sshr x (maybe_uextend (imm12_from_value y)))))
   (rv_sraiw x y))
diff --git a/cranelift/codegen/src/machinst/vcode.rs b/cranelift/codegen/src/machinst/vcode.rs
index c743fcd20..90bb2a286 100644
--- a/cranelift/codegen/src/machinst/vcode.rs
+++ b/cranelift/codegen/src/machinst/vcode.rs
@@ -1738,6 +1738,15 @@ impl VCodeConstants {
     pub fn get(&self, c: VCodeConstant) -> &VCodeConstantData {
         &self.constants[c]
     }
+
+    /// Checks if the given [VCodeConstantData] is registered as
+    /// used by the pool.
+    pub fn pool_uses(&self, constant: &VCodeConstantData) -> bool {
+        match constant {
+            VCodeConstantData::Pool(c, _) => self.pool_uses.contains_key(c),
+            _ => false,
+        }
+    }
 }
 
 /// A use of a constant by one or more VCode instructions; see [VCodeConstants].
diff --git a/cranelift/codegen/src/opts/bitops.isle b/cranelift/codegen/src/opts/bitops.isle
index 018db93b6..6c89e0f07 100644
--- a/cranelift/codegen/src/opts/bitops.isle
+++ b/cranelift/codegen/src/opts/bitops.isle
@@ -98,11 +98,11 @@
 ;; (x | -x) sets the sign bit to 1 if x is nonzero, and 0 if x is zero. sshr propagates
 ;; the sign bit to the rest of the value.
 (rule (simplify (sshr ty (bor ty x (ineg ty x)) (iconst ty (u64_from_imm64 shift_amt))))
-      (if-let $true (u64_eq shift_amt (u64_sub (ty_bits ty) 1)))
+      (if-let $true (u64_eq shift_amt (ty_shift_mask ty)))
       (bmask ty x))
 
 (rule (simplify (sshr ty (bor ty (ineg ty x) x) (iconst ty (u64_from_imm64 shift_amt))))
-      (if-let $true (u64_eq shift_amt (u64_sub (ty_bits ty) 1)))
+      (if-let $true (u64_eq shift_amt (ty_shift_mask ty)))
       (bmask ty x))
 
 ;; Matches any expressions that preserve "truthiness".
diff --git a/cranelift/codegen/src/opts/shifts.isle b/cranelift/codegen/src/opts/shifts.isle
index 1b33d89d8..acf53946c 100644
--- a/cranelift/codegen/src/opts/shifts.isle
+++ b/cranelift/codegen/src/opts/shifts.isle
@@ -100,7 +100,7 @@
 
 ;; ineg(ushr(x, k)) == sshr(x, k) when k == ty_bits - 1.
 (rule (simplify (ineg ty (ushr ty x sconst @ (iconst ty (u64_from_imm64 shift_amt)))))
-      (if-let $true (u64_eq shift_amt (u64_sub (ty_bits ty) 1)))
+      (if-let $true (u64_eq shift_amt (ty_shift_mask ty)))
       (sshr ty x sconst))
 
 ;; Shifts and rotates allow a different type for the shift amount, so we
@@ -144,3 +144,127 @@
 (rule (simplify (sshr ty x (iconcat _ y _))) (sshr ty x y))
 (rule (simplify (rotr ty x (iconcat _ y _))) (rotr ty x y))
 (rule (simplify (rotl ty x (iconcat _ y _))) (rotl ty x y))
+
+;; Try to combine the shift amount from multiple consecutive shifts
+;; This only works if the shift amount remains smaller than the bit
+;; width of the type.
+;;
+;; (ishl (ishl x k1) k2) == (ishl x (add k1 k2)) if shift_mask(k1) + shift_mask(k2) < ty_bits
+;; (ushr (ushr x k1) k2) == (ushr x (add k1 k2)) if shift_mask(k1) + shift_mask(k2) < ty_bits
+;; (sshr (sshr x k1) k2) == (sshr x (add k1 k2)) if shift_mask(k1) + shift_mask(k2) < ty_bits
+(rule (simplify (ishl ty
+                      (ishl ty x (iconst kty (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_lt shift_amt (ty_bits_u64 (lane_type ty))))
+      (ishl ty x (iconst_u64 kty shift_amt)))
+
+(rule (simplify (ushr ty
+                      (ushr ty x (iconst kty (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_lt shift_amt (ty_bits_u64 (lane_type ty))))
+      (ushr ty x (iconst_u64 kty shift_amt)))
+
+(rule (simplify (sshr ty
+                      (sshr ty x (iconst kty (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_lt shift_amt (ty_bits_u64 (lane_type ty))))
+      (sshr ty x (iconst_u64 kty shift_amt)))
+
+;; Simliarly, if the shift amount overflows the type, then we can turn
+;; it into a 0
+;;
+;; (ishl (ishl x k1) k2) == 0 if shift_mask(k1) + shift_mask(k2) >= ty_bits
+;; (ushr (ushr x k1) k2) == 0 if shift_mask(k1) + shift_mask(k2) >= ty_bits
+(rule (simplify (ishl ty
+                      (ishl ty x (iconst _ (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_le (ty_bits_u64 ty) shift_amt))
+      (subsume (iconst_u64 ty 0)))
+
+(rule (simplify (ushr ty
+                      (ushr ty x (iconst _ (u64_from_imm64 k1)))
+                      (iconst _ (u64_from_imm64 k2))))
+      (if-let shift_amt (u64_add
+                              (u64_and k1 (ty_shift_mask ty))
+                              (u64_and k2 (ty_shift_mask ty))))
+      (if-let $true (u64_le (ty_bits_u64 ty) shift_amt))
+      (subsume (iconst_u64 ty 0)))
+
+;; (rotl (rotr x y) y) == x
+;; (rotr (rotl x y) y) == x
+(rule (simplify (rotl ty (rotr ty x y) y)) (subsume x))
+(rule (simplify (rotr ty (rotl ty x y) y)) (subsume x))
+
+;; Emits an iadd for two values. If they have different types
+;; then the smaller type is zero extended to the larger type.
+(decl iadd_uextend (Value Value) Value)
+(rule 1 (iadd_uextend x @ (value_type ty) y @ (value_type ty))
+      (iadd ty x y))
+(rule 2 (iadd_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 x_ty) (ty_bits_u64 y_ty)))
+      (iadd y_ty (uextend y_ty x) y))
+(rule 3 (iadd_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 y_ty) (ty_bits_u64 x_ty)))
+      (iadd x_ty x (uextend x_ty y)))
+
+;; Emits an isub for two values. If they have different types
+;; then the smaller type is zero extended to the larger type.
+(decl isub_uextend (Value Value) Value)
+(rule 1 (isub_uextend x @ (value_type ty) y @ (value_type ty))
+      (isub ty x y))
+(rule 2 (isub_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 x_ty) (ty_bits_u64 y_ty)))
+      (isub y_ty (uextend y_ty x) y))
+(rule 3 (isub_uextend x @ (value_type x_ty) y @ (value_type y_ty))
+      (if-let $true (u64_lt (ty_bits_u64 y_ty) (ty_bits_u64 x_ty)))
+      (isub x_ty x (uextend x_ty y)))
+
+;; Try to group constants together so that other cprop rules can optimize them.
+;;
+;; (rotr (rotr x y) z) == (rotr x (iadd y z))
+;; (rotl (rotl x y) z) == (rotl x (iadd y z))
+;; (rotr (rotl x y) z) == (rotr x (isub y z))
+;; (rotl (rotr x y) z) == (rotl x (isub y z))
+;;
+;; if x or z are constants
+(rule (simplify (rotl ty (rotl ty x y @ (iconst _ _)) z)) (rotl ty x (iadd_uextend y z)))
+(rule (simplify (rotl ty (rotl ty x y) z @ (iconst _ _))) (rotl ty x (iadd_uextend y z)))
+(rule (simplify (rotr ty (rotr ty x y @ (iconst _ _)) z)) (rotr ty x (iadd_uextend y z)))
+(rule (simplify (rotr ty (rotr ty x y) z @ (iconst _ _))) (rotr ty x (iadd_uextend y z)))
+
+(rule (simplify (rotr ty (rotl ty x y @ (iconst _ _)) z)) (rotl ty x (isub_uextend y z)))
+(rule (simplify (rotr ty (rotl ty x y) z @ (iconst _ _))) (rotl ty x (isub_uextend y z)))
+(rule (simplify (rotl ty (rotr ty x y @ (iconst _ _)) z)) (rotr ty x (isub_uextend y z)))
+(rule (simplify (rotl ty (rotr ty x y) z @ (iconst _ _))) (rotr ty x (isub_uextend y z)))
+
+;; Similarly to the rules above, if y and z have the same type, we should emit
+;; an iadd or isub instead. In some backends this is cheaper than a rotate.
+;;
+;; If they have different types we end up in a situation where we have to insert
+;; and additional extend and that transformation is not universally beneficial.
+;;
+;; (rotr (rotr x y) z) == (rotr x (iadd y z))
+;; (rotl (rotl x y) z) == (rotl x (iadd y z))
+;; (rotr (rotl x y) z) == (rotl x (isub y z))
+;; (rotl (rotr x y) z) == (rotr x (isub y z))
+(rule (simplify (rotr ty (rotr ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotr ty x (iadd_uextend y z)))
+(rule (simplify (rotl ty (rotl ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotl ty x (iadd_uextend y z)))
+
+(rule (simplify (rotr ty (rotl ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotl ty x (isub_uextend y z)))
+(rule (simplify (rotl ty (rotr ty x y @ (value_type kty)) z @ (value_type kty)))
+      (rotr ty x (isub_uextend y z)))
diff --git a/cranelift/codegen/src/prelude.isle b/cranelift/codegen/src/prelude.isle
index eb03dd24a..c61876b8d 100644
--- a/cranelift/codegen/src/prelude.isle
+++ b/cranelift/codegen/src/prelude.isle
@@ -323,6 +323,10 @@
 (decl pure partial ty_half_width (Type) Type)
 (extern constructor ty_half_width ty_half_width)
 
+;; Generate a mask for the maximum shift amount for a given type. i.e 31 for I32.
+(decl pure ty_shift_mask (Type) u64)
+(rule (ty_shift_mask ty) (u64_sub (ty_bits (lane_type ty)) 1))
+
 ;; Compare two types for equality.
 (decl pure ty_equal (Type Type) bool)
 (extern constructor ty_equal ty_equal)
diff --git a/cranelift/codegen/src/prelude_opt.isle b/cranelift/codegen/src/prelude_opt.isle
index 099930f5b..921c7c65b 100644
--- a/cranelift/codegen/src/prelude_opt.isle
+++ b/cranelift/codegen/src/prelude_opt.isle
@@ -43,3 +43,16 @@
 ;; answer".
 (decl subsume (Value) Value)
 (extern constructor subsume subsume)
+
+;;;;; constructors ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;
+
+(decl iconst_u64 (Type u64) Value)
+
+;; Use a single iconst for types that fit in 64 bits.
+(rule 0 (iconst_u64 (ty_int_ref_scalar_64 ty) val)
+    (if-let $true (u64_le val (ty_umax ty)))
+    (iconst ty (imm64_masked ty val)))
+
+;; For i128 types use a iconst, but zero extend it to i128.
+(rule 1 (iconst_u64 $I128 val)
+    (uextend $I128 (iconst $I64 (imm64_masked $I64 val))))
diff --git a/crates/c-api/include/wasmtime.h b/crates/c-api/include/wasmtime.h
index 44e4e3d2c..ce7d46e0f 100644
--- a/crates/c-api/include/wasmtime.h
+++ b/crates/c-api/include/wasmtime.h
@@ -181,6 +181,7 @@
 #define WASMTIME_API_H
 
 #include <wasi.h>
+// IWYU pragma: begin_exports
 #include <wasmtime/config.h>
 #include <wasmtime/engine.h>
 #include <wasmtime/error.h>
@@ -196,6 +197,7 @@
 #include <wasmtime/trap.h>
 #include <wasmtime/val.h>
 #include <wasmtime/async.h>
+// IWYU pragma: end_exports
 
 /**
  * \brief Wasmtime version string.
diff --git a/crates/cli-flags/src/lib.rs b/crates/cli-flags/src/lib.rs
index 03900707a..d785435b6 100644
--- a/crates/cli-flags/src/lib.rs
+++ b/crates/cli-flags/src/lib.rs
@@ -65,6 +65,14 @@ wasmtime_option_group! {
         /// Enable the pooling allocator, in place of the on-demand allocator.
         pub pooling_allocator: Option<bool>,
 
+        /// How many bytes to keep resident between instantiations for the
+        /// pooling allocator in linear memories.
+        pub pooling_memory_keep_resident: Option<usize>,
+
+        /// How many bytes to keep resident between instantiations for the
+        /// pooling allocator in tables.
+        pub pooling_table_keep_resident: Option<usize>,
+
         /// Configure attempting to initialize linear memory via a
         /// copy-on-write mapping (default: yes)
         pub memory_init_cow: Option<bool>,
@@ -500,7 +508,14 @@ impl CommonOptions {
             ["pooling-allocator" : self.opts.pooling_allocator]
             enable => {
                 if enable {
-                    config.allocation_strategy(wasmtime::InstanceAllocationStrategy::pooling());
+                    let mut cfg = wasmtime::PoolingAllocationConfig::default();
+                    if let Some(size) = self.opts.pooling_memory_keep_resident {
+                        cfg.linear_memory_keep_resident(size);
+                    }
+                    if let Some(size) = self.opts.pooling_table_keep_resident {
+                        cfg.table_keep_resident(size);
+                    }
+                    config.allocation_strategy(wasmtime::InstanceAllocationStrategy::Pooling(cfg));
                 }
             },
             true => err,
diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs
index 5b4909b48..517b5d1b9 100644
--- a/crates/jit/src/code_memory.rs
+++ b/crates/jit/src/code_memory.rs
@@ -149,22 +149,26 @@ impl CodeMemory {
     }
 
     /// Returns a reference to the underlying `MmapVec` this memory owns.
+    #[inline]
     pub fn mmap(&self) -> &MmapVec {
         &self.mmap
     }
 
     /// Returns the contents of the text section of the ELF executable this
     /// represents.
+    #[inline]
     pub fn text(&self) -> &[u8] {
         &self.mmap[self.text.clone()]
     }
 
     /// Returns the contents of the `ELF_WASMTIME_DWARF` section.
+    #[inline]
     pub fn dwarf(&self) -> &[u8] {
         &self.mmap[self.dwarf.clone()]
     }
 
     /// Returns the data in the `ELF_NAME_DATA` section.
+    #[inline]
     pub fn func_name_data(&self) -> &[u8] {
         &self.mmap[self.func_name_data.clone()]
     }
@@ -174,24 +178,28 @@ impl CodeMemory {
     ///
     /// This is used for initialization of memories and all data ranges stored
     /// in a `Module` are relative to the slice returned here.
+    #[inline]
     pub fn wasm_data(&self) -> &[u8] {
         &self.mmap[self.wasm_data.clone()]
     }
 
     /// Returns the encoded address map section used to pass to
     /// `wasmtime_environ::lookup_file_pos`.
+    #[inline]
     pub fn address_map_data(&self) -> &[u8] {
         &self.mmap[self.address_map_data.clone()]
     }
 
     /// Returns the contents of the `ELF_WASMTIME_INFO` section, or an empty
     /// slice if it wasn't found.
+    #[inline]
     pub fn wasmtime_info(&self) -> &[u8] {
         &self.mmap[self.info_data.clone()]
     }
 
     /// Returns the contents of the `ELF_WASMTIME_TRAPS` section, or an empty
     /// slice if it wasn't found.
+    #[inline]
     pub fn trap_data(&self) -> &[u8] {
         &self.mmap[self.trap_data.clone()]
     }
diff --git a/crates/runtime/src/mmap.rs b/crates/runtime/src/mmap.rs
index 4f9f4c64d..341cd4dee 100644
--- a/crates/runtime/src/mmap.rs
+++ b/crates/runtime/src/mmap.rs
@@ -146,11 +146,13 @@ impl Mmap {
     }
 
     /// Return the allocated memory as a pointer to u8.
+    #[inline]
     pub fn as_ptr(&self) -> *const u8 {
         self.sys.as_ptr()
     }
 
     /// Return the allocated memory as a mutable pointer to u8.
+    #[inline]
     pub fn as_mut_ptr(&mut self) -> *mut u8 {
         self.sys.as_mut_ptr()
     }
@@ -159,6 +161,7 @@ impl Mmap {
     ///
     /// This is the byte length of this entire mapping which includes both
     /// addressible and non-addressible memory.
+    #[inline]
     pub fn len(&self) -> usize {
         self.sys.len()
     }
diff --git a/crates/runtime/src/mmap/unix.rs b/crates/runtime/src/mmap/unix.rs
index 3227ec3a0..97de914da 100644
--- a/crates/runtime/src/mmap/unix.rs
+++ b/crates/runtime/src/mmap/unix.rs
@@ -84,14 +84,17 @@ impl Mmap {
         Ok(())
     }
 
+    #[inline]
     pub fn as_ptr(&self) -> *const u8 {
         self.memory.as_ptr() as *const u8
     }
 
+    #[inline]
     pub fn as_mut_ptr(&mut self) -> *mut u8 {
         self.memory.as_ptr().cast()
     }
 
+    #[inline]
     pub fn len(&self) -> usize {
         unsafe { (*self.memory.as_ptr()).len() }
     }
diff --git a/crates/runtime/src/mmap/windows.rs b/crates/runtime/src/mmap/windows.rs
index 5ca208b33..4a8396c44 100644
--- a/crates/runtime/src/mmap/windows.rs
+++ b/crates/runtime/src/mmap/windows.rs
@@ -147,14 +147,17 @@ impl Mmap {
         Ok(())
     }
 
+    #[inline]
     pub fn as_ptr(&self) -> *const u8 {
         self.memory.as_ptr() as *const u8
     }
 
+    #[inline]
     pub fn as_mut_ptr(&mut self) -> *mut u8 {
         self.memory.as_ptr().cast()
     }
 
+    #[inline]
     pub fn len(&self) -> usize {
         unsafe { (*self.memory.as_ptr()).len() }
     }
diff --git a/crates/wasmtime/src/compiler.rs b/crates/wasmtime/src/compiler.rs
index 33856f4b2..5022ef9da 100644
--- a/crates/wasmtime/src/compiler.rs
+++ b/crates/wasmtime/src/compiler.rs
@@ -28,7 +28,7 @@ use std::collections::{btree_map, BTreeMap, BTreeSet};
 use std::{any::Any, collections::HashMap};
 use wasmtime_environ::{
     Compiler, DefinedFuncIndex, FuncIndex, FunctionBodyData, ModuleTranslation, ModuleType,
-    ModuleTypes, PrimaryMap, SignatureIndex, StaticModuleIndex, Tunables, WasmFunctionInfo,
+    ModuleTypes, PrimaryMap, SignatureIndex, StaticModuleIndex, WasmFunctionInfo,
 };
 use wasmtime_jit::{CompiledFunctionInfo, CompiledModuleInfo};
 
@@ -447,8 +447,7 @@ impl FunctionIndices {
     pub fn link_and_append_code<'a>(
         mut self,
         mut obj: object::write::Object<'static>,
-        tunables: &'a Tunables,
-        compiler: &dyn Compiler,
+        engine: &'a Engine,
         compiled_funcs: Vec<(String, Box<dyn Any + Send>)>,
         translations: PrimaryMap<StaticModuleIndex, ModuleTranslation<'_>>,
     ) -> Result<(wasmtime_jit::ObjectBuilder<'a>, Artifacts)> {
@@ -457,6 +456,8 @@ impl FunctionIndices {
         // The result is a vector parallel to `compiled_funcs` where
         // `symbol_ids_and_locs[i]` is the symbol ID and function location of
         // `compiled_funcs[i]`.
+        let compiler = engine.compiler();
+        let tunables = &engine.config().tunables;
         let symbol_ids_and_locs = compiler.append_code(
             &mut obj,
             &compiled_funcs,
@@ -559,7 +560,22 @@ impl FunctionIndices {
 
         artifacts.modules = translations
             .into_iter()
-            .map(|(module, translation)| {
+            .map(|(module, mut translation)| {
+                // If configured attempt to use static memory initialization which
+                // can either at runtime be implemented as a single memcpy to
+                // initialize memory or otherwise enabling virtual-memory-tricks
+                // such as mmap'ing from a file to get copy-on-write.
+                if engine.config().memory_init_cow {
+                    let align = compiler.page_size_align();
+                    let max_always_allowed = engine.config().memory_guaranteed_dense_image_size;
+                    translation.try_static_init(align, max_always_allowed);
+                }
+
+                // Attempt to convert table initializer segments to
+                // FuncTable representation where possible, to enable
+                // table lazy init.
+                translation.try_func_table_init();
+
                 let funcs: PrimaryMap<DefinedFuncIndex, CompiledFunctionInfo> =
                     wasm_functions_for_module(&mut wasm_functions, module)
                         .map(|(key, wasm_func_index)| {
diff --git a/crates/wasmtime/src/component/component.rs b/crates/wasmtime/src/component/component.rs
index 013f93bd7..a57165ed4 100644
--- a/crates/wasmtime/src/component/component.rs
+++ b/crates/wasmtime/src/component/component.rs
@@ -206,8 +206,7 @@ impl Component {
 
         let (mut object, compilation_artifacts) = function_indices.link_and_append_code(
             object,
-            &engine.config().tunables,
-            compiler,
+            engine,
             compiled_funcs,
             module_translations,
         )?;
@@ -473,3 +472,34 @@ impl ComponentRuntimeInfo for ComponentInner {
         }
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::component::Component;
+    use crate::{Config, Engine};
+    use wasmtime_environ::MemoryInitialization;
+
+    #[test]
+    fn cow_on_by_default() {
+        let mut config = Config::new();
+        config.wasm_component_model(true);
+        let engine = Engine::new(&config).unwrap();
+        let component = Component::new(
+            &engine,
+            r#"
+                (component
+                    (core module
+                        (memory 1)
+                        (data (i32.const 100) "abcd")
+                    )
+                )
+            "#,
+        )
+        .unwrap();
+
+        for (_, module) in component.inner.static_modules.iter() {
+            let init = &module.env_module().memory_initialization;
+            assert!(matches!(init, MemoryInitialization::Static { .. }));
+        }
+    }
+}
diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs
index 38a51fcd7..a5553ff94 100644
--- a/crates/wasmtime/src/module.rs
+++ b/crates/wasmtime/src/module.rs
@@ -402,7 +402,6 @@ impl Module {
         use crate::compiler::CompileInputs;
 
         let tunables = &engine.config().tunables;
-        let compiler = engine.compiler();
 
         // First a `ModuleEnvironment` is created which records type information
         // about the wasm module. This is where the WebAssembly is parsed and
@@ -437,25 +436,9 @@ impl Module {
         engine.append_compiler_info(&mut object);
         engine.append_bti(&mut object);
 
-        // If configured attempt to use static memory initialization which
-        // can either at runtime be implemented as a single memcpy to
-        // initialize memory or otherwise enabling virtual-memory-tricks
-        // such as mmap'ing from a file to get copy-on-write.
-        if engine.config().memory_init_cow {
-            let align = engine.compiler().page_size_align();
-            let max_always_allowed = engine.config().memory_guaranteed_dense_image_size;
-            translation.try_static_init(align, max_always_allowed);
-        }
-
-        // Attempt to convert table initializer segments to
-        // FuncTable representation where possible, to enable
-        // table lazy init.
-        translation.try_func_table_init();
-
         let (mut object, compilation_artifacts) = function_indices.link_and_append_code(
             object,
-            tunables,
-            compiler,
+            engine,
             compiled_funcs,
             std::iter::once(translation).collect(),
         )?;
@@ -1361,3 +1344,27 @@ fn memory_images(engine: &Engine, module: &CompiledModule) -> Result<Option<Modu
     };
     ModuleMemoryImages::new(module.module(), module.code_memory().wasm_data(), mmap)
 }
+
+#[cfg(test)]
+mod tests {
+    use crate::{Engine, Module};
+    use wasmtime_environ::MemoryInitialization;
+
+    #[test]
+    fn cow_on_by_default() {
+        let engine = Engine::default();
+        let module = Module::new(
+            &engine,
+            r#"
+                (module
+                    (memory 1)
+                    (data (i32.const 100) "abcd")
+                )
+            "#,
+        )
+        .unwrap();
+
+        let init = &module.env_module().memory_initialization;
+        assert!(matches!(init, MemoryInitialization::Static { .. }));
+    }
+}
diff --git a/crates/wast/Cargo.toml b/crates/wast/Cargo.toml
index 24b690427..b26451139 100644
--- a/crates/wast/Cargo.toml
+++ b/crates/wast/Cargo.toml
@@ -11,7 +11,7 @@ edition.workspace = true
 
 [dependencies]
 anyhow = { workspace = true }
-wasmtime = { workspace = true, features = ['cranelift'] }
+wasmtime = { workspace = true, features = ['cranelift', 'wat'] }
 wast = { workspace = true }
 log = { workspace = true }
 
diff --git a/src/commands/serve.rs b/src/commands/serve.rs
index 31276a1aa..927cc755b 100644
--- a/src/commands/serve.rs
+++ b/src/commands/serve.rs
@@ -373,12 +373,16 @@ impl hyper::service::Service<Request> for ProxyHandler {
                 wasmtime_wasi_http::proxy::Proxy::instantiate_pre(&mut store, &inner.instance_pre)
                     .await?;
 
-            proxy
+            if let Err(e) = proxy
                 .wasi_http_incoming_handler()
                 .call_handle(store, req, out)
-                .await?;
+                .await
+            {
+                log::error!("[{req_id}] :: {:#?}", e);
+                return Err(e);
+            }
 
-            Ok::<_, anyhow::Error>(())
+            Ok(())
         });
 
         Box::pin(async move {
diff --git a/winch/codegen/src/isa/x64/asm.rs b/winch/codegen/src/isa/x64/asm.rs
index 98f7723c6..bd791784e 100644
--- a/winch/codegen/src/isa/x64/asm.rs
+++ b/winch/codegen/src/isa/x64/asm.rs
@@ -202,8 +202,14 @@ impl Assembler {
                 // instrunction.
                 let constant_data = pool.get(*c);
                 let data = VCodeConstantData::Pool(*c, constant_data.clone());
+                // If the constaant data is not marked as used, it will be
+                // inserted, therefore, it needs to be registered.
+                let needs_registration = !constants.pool_uses(&data);
                 let constant = constants.insert(VCodeConstantData::Pool(*c, constant_data.clone()));
-                buffer.register_constant(&constant, &data);
+
+                if needs_registration {
+                    buffer.register_constant(&constant, &data);
+                }
                 SyntheticAmode::ConstantOffset(constant)
             }
         }


Test patch:
diff --git a/cranelift/filetests/filetests/egraph/shifts.clif b/cranelift/filetests/filetests/egraph/shifts.clif
index 7852ada12..dbe984c84 100644
--- a/cranelift/filetests/filetests/egraph/shifts.clif
+++ b/cranelift/filetests/filetests/egraph/shifts.clif
@@ -524,3 +524,237 @@ block0(v0: i8, v1: i16, v2: i16):
 
 ; check: v5 = rotl v0, v1
 ; check: return v5
+
+function %ishl_prop_overflow(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 7
+    v2 = iconst.i8 7
+    v3 = ishl.i8 v0, v1
+    v4 = ishl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 0
+; check: return v5  ; v5 = 0
+
+function %ishl_prop_type_diff(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i16 7
+    v2 = iconst.i16 7
+    v3 = ishl.i8 v0, v1
+    v4 = ishl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 0
+; check: return v5  ; v5 = 0
+
+function %ushr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = ushr.i8 v0, v1
+    v4 = ushr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 2
+; check: v6 = ushr v0, v5  ; v5 = 2
+; check: return v6
+
+function %ushr_prop_overflow(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 7
+    v2 = iconst.i8 7
+    v3 = ushr.i8 v0, v1
+    v4 = ushr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 0
+; check: return v5  ; v5 = 0
+
+
+function %sshr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = sshr.i8 v0, v1
+    v4 = sshr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iconst.i8 2
+; check: v6 = sshr v0, v5  ; v5 = 2
+; check: return v6
+
+function %i128_ushr_overflow_becomes_0(i128) -> i128 {
+block0(v0: i128):
+    v1 = iconst.i16 200
+    v2 = ushr v0, v1
+    v3 = ushr v2, v1
+    return v3
+}
+
+; check: v4 = iconst.i64 0
+; nextln: v5 = uextend.i128 v4  ; v4 = 0
+; nextln: return v5
+
+
+function %i128_ishl_overflow_becomes_0(i128) -> i128 {
+block0(v0: i128):
+    v1 = iconst.i16 200
+    v2 = ishl v0, v1
+    v3 = ishl v2, v1
+    return v3
+}
+
+; check: v4 = iconst.i64 0
+; nextln: v5 = uextend.i128 v4  ; v4 = 0
+; nextln: return v5
+
+function %simd_shift_does_not_panic(i8x16) -> i8x16 {
+block0(v0: i8x16):
+    v1 = iconst.i64 0
+    v2 = ushr v0, v1
+    v3 = ushr v2, v1
+    return v3
+}
+
+; check: v1 = iconst.i64 0
+; nextln: v2 = ushr v0, v1  ; v1 = 0
+; check: return v2
+
+function %merges_shift_amount_based_on_lane_type(i8x16) -> i8x16 {
+block0(v0: i8x16):
+    v1 = iconst.i16 30
+    v2 = iconst.i16 3
+
+    v3 = sshr v0, v1
+    v4 = sshr v3, v2
+    v5 = sshr v4, v2
+    return v5
+}
+
+; check: v1 = iconst.i16 30
+; nextln: v3 = sshr v0, v1  ; v1 = 30
+; nextln: v6 = iconst.i16 6
+; nextln: v7 = sshr v3, v6  ; v6 = 6
+; check: return v7
+
+
+function %rotr_rotr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = rotr.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v11 = iconst.i8 2
+; check: v13 = rotr v0, v11  ; v11 = 2
+; check: return v13
+
+function %rotr_rotr_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotr.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iadd v1, v2
+; check: v6 = rotr v0, v5
+; check: return v6
+
+function %rotl_rotl_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 1
+    v2 = iconst.i8 1
+    v3 = rotl.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v11 = iconst.i8 2
+; check: v13 = rotl v0, v11  ; v11 = 2
+; check: return v13
+
+function %rotl_rotl_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotl.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = iadd v1, v2
+; check: v6 = rotl v0, v5
+; check: return v6
+
+
+function %rotl_rotr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 2
+    v2 = iconst.i8 1
+    v3 = rotr.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v2 = iconst.i8 1
+; check: v21 = rotr v0, v2  ; v2 = 1
+; check: return v21
+
+function %rotl_rotr_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotr.i8 v0, v1
+    v4 = rotl.i8 v3, v2
+    return v4
+}
+
+; check: v5 = isub v1, v2
+; check: v6 = rotr v0, v5
+; check: return v6
+
+function %rotl_rotr_prop(i8) -> i8 {
+block0(v0: i8):
+    v1 = iconst.i8 2
+    v2 = iconst.i8 1
+    v3 = rotl.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v2 = iconst.i8 1
+; check: v21 = rotl v0, v2  ; v2 = 1
+; check: return v21
+
+function %rotl_rotr_add(i8, i8, i8) -> i8 {
+block0(v0: i8, v1: i8, v2: i8):
+    v3 = rotl.i8 v0, v1
+    v4 = rotr.i8 v3, v2
+    return v4
+}
+
+; check: v5 = isub v1, v2
+; check: v6 = rotl v0, v5
+; check: return v6
+
+function %rotl_rotr_subsume(i8, i8) -> i8 {
+block0(v0: i8, v1: i8):
+    v2 = rotl.i8 v0, v1
+    v3 = rotr.i8 v2, v1
+    return v3
+}
+
+; check: return v0
+
+
+function %rotr_rotl_subsume(i8, i8) -> i8 {
+block0(v0: i8, v1: i8):
+    v2 = rotr.i8 v0, v1
+    v3 = rotl.i8 v2, v1
+    return v3
+}
+
+; check: return v0
diff --git a/cranelift/filetests/filetests/isa/aarch64/condbr.clif b/cranelift/filetests/filetests/isa/aarch64/condbr.clif
index f0661556d..13d59de14 100644
--- a/cranelift/filetests/filetests/isa/aarch64/condbr.clif
+++ b/cranelift/filetests/filetests/isa/aarch64/condbr.clif
@@ -733,3 +733,183 @@ block1:
 ; block1: ; offset 0x1c
 ;   ret
 
+function %tbnz_i8(i8) {
+block0(v0: i8):
+  v1 = band_imm v0, 0x10
+  brif v1, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbnz x0, #4, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbnz w0, #4, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+function %tbz_i16(i16) {
+block0(v0: i16):
+  v1 = band_imm v0, 0x1000
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbz x0, #12, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbz w0, #0xc, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+function %tbnz_i32(i32) {
+block0(v0: i32):
+  v1 = band_imm v0, 0x10000
+  brif v1, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbnz x0, #16, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbnz w0, #0x10, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+
+function %tbz_i64(i64) {
+block0(v0: i64):
+  v1 = band_imm v0, 0x1_00000000
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   tbz x0, #32, label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   tbz x0, #0x20, #8
+; block1: ; offset 0x4
+;   ret
+; block2: ; offset 0x8
+;   ret
+
+function %not_tbz1(i8) {
+block0(v0: i8):
+  v1 = band_imm v0, 0x100
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   movz w4, #0
+;   and w4, w0, w4
+;   uxtb w4, w4
+;   subs wzr, w4, #0
+;   b.eq label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   mov w4, #0
+;   and w4, w0, w4
+;   uxtb w4, w4
+;   cmp w4, #0
+;   b.eq #0x18
+; block1: ; offset 0x14
+;   ret
+; block2: ; offset 0x18
+;   ret
+
+function %not_tbz2(i8) {
+block0(v0: i8):
+  v1 = band_imm v0, 0x3
+  v2 = icmp_imm eq v1, 0
+  brif v2, block1, block2
+
+block1:
+  return
+block2:
+  return
+}
+
+; VCode:
+; block0:
+;   and w3, w0, #3
+;   uxtb w3, w3
+;   subs wzr, w3, #0
+;   b.eq label2 ; b label1
+; block1:
+;   ret
+; block2:
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   and w3, w0, #3
+;   uxtb w3, w3
+;   cmp w3, #0
+;   b.eq #0x14
+; block1: ; offset 0x10
+;   ret
+; block2: ; offset 0x14
+;   ret
diff --git a/tests/misc_testsuite/winch/local_get.wast b/tests/misc_testsuite/winch/local_get.wast
new file mode 100644
index 000000000..16e06113d
--- /dev/null
+++ b/tests/misc_testsuite/winch/local_get.wast
@@ -0,0 +1,103 @@
+;; Test `local.get` operator
+
+(module
+  ;; Typing
+
+  (func (export "type-local-i32") (result i32) (local i32) (local.get 0))
+  (func (export "type-local-i64") (result i64) (local i64) (local.get 0))
+  (func (export "type-local-f32") (result f32) (local f32) (local.get 0))
+  (func (export "type-local-f64") (result f64) (local f64) (local.get 0))
+
+  (func (export "type-param-i32") (param i32) (result i32) (local.get 0))
+  (func (export "type-param-i64") (param i64) (result i64) (local.get 0))
+  (func (export "type-param-f32") (param f32) (result f32) (local.get 0))
+  (func (export "type-param-f64") (param f64) (result f64) (local.get 0))
+
+  (func (export "type-mixed") (param i64 f32 f64 i32 i32)
+    (local f32 i64 i64 f64)
+    (drop (i64.eqz (local.get 0)))
+    (drop (f32.neg (local.get 1)))
+    (drop (f64.neg (local.get 2)))
+    (drop (i32.eqz (local.get 3)))
+    (drop (i32.eqz (local.get 4)))
+    (drop (f32.neg (local.get 5)))
+    (drop (i64.eqz (local.get 6)))
+    (drop (i64.eqz (local.get 7)))
+    (drop (f64.neg (local.get 8)))
+  )
+
+
+  ;; As parameter of control constructs and instructions
+
+  (func (export "as-block-value") (param i32) (result i32)
+    (block (result i32) (local.get 0))
+  )
+  (func (export "as-loop-value") (param i32) (result i32)
+    (loop (result i32) (local.get 0))
+  )
+  (func (export "as-br-value") (param i32) (result i32)
+    (block (result i32) (br 0 (local.get 0)))
+  )
+  (func (export "as-br_if-value") (param i32) (result i32)
+    (block $l0 (result i32) (br_if $l0 (local.get 0) (i32.const 1)))
+  )
+
+  (func (export "as-br_if-value-cond") (param i32) (result i32)
+    (block (result i32)
+      (br_if 0 (local.get 0) (local.get 0))
+    )
+  )
+  (func (export "as-br_table-value") (param i32) (result i32)
+    (block
+      (block
+        (block
+          (br_table 0 1 2 (local.get 0))
+          (return (i32.const 0))
+        )
+        (return (i32.const 1))
+      )
+      (return (i32.const 2))
+    )
+    (i32.const 3)
+  )
+
+  (func (export "as-return-value") (param i32) (result i32)
+    (return (local.get 0))
+  )
+
+  (func (export "as-if-then") (param i32) (result i32)
+    (if (result i32) (local.get 0) (then (local.get 0)) (else (i32.const 0)))
+  )
+  (func (export "as-if-else") (param i32) (result i32)
+    (if (result i32) (local.get 0) (then (i32.const 1)) (else (local.get 0)))
+  )
+)
+
+(assert_return (invoke "type-local-i32") (i32.const 0))
+(assert_return (invoke "type-local-i64") (i64.const 0))
+(assert_return (invoke "type-local-f32") (f32.const 0))
+(assert_return (invoke "type-local-f64") (f64.const 0))
+
+(assert_return (invoke "type-param-i32" (i32.const 2)) (i32.const 2))
+(assert_return (invoke "type-param-i64" (i64.const 3)) (i64.const 3))
+(assert_return (invoke "type-param-f32" (f32.const 4.4)) (f32.const 4.4))
+(assert_return (invoke "type-param-f64" (f64.const 5.5)) (f64.const 5.5))
+
+(assert_return (invoke "as-block-value" (i32.const 6)) (i32.const 6))
+(assert_return (invoke "as-loop-value" (i32.const 7)) (i32.const 7))
+
+(assert_return (invoke "as-br-value" (i32.const 8)) (i32.const 8))
+(assert_return (invoke "as-br_if-value" (i32.const 9)) (i32.const 9))
+(assert_return (invoke "as-br_if-value-cond" (i32.const 10)) (i32.const 10))
+(assert_return (invoke "as-br_table-value" (i32.const 1)) (i32.const 2))
+
+(assert_return (invoke "as-return-value" (i32.const 0)) (i32.const 0))
+
+(assert_return (invoke "as-if-then" (i32.const 1)) (i32.const 1))
+(assert_return (invoke "as-if-else" (i32.const 0)) (i32.const 0))
+
+(assert_return
+  (invoke "type-mixed"
+    (i64.const 1) (f32.const 2.2) (f64.const 3.3) (i32.const 4) (i32.const 5)
+  )
+)
diff --git a/tests/misc_testsuite/winch/local_set.wast b/tests/misc_testsuite/winch/local_set.wast
new file mode 100644
index 000000000..f368f7bf3
--- /dev/null
+++ b/tests/misc_testsuite/winch/local_set.wast
@@ -0,0 +1,97 @@
+;; Test `local.set` operator
+
+(module
+  ;; Typing
+
+  (func (export "type-local-i32") (local i32) (local.set 0 (i32.const 0)))
+  (func (export "type-local-i64") (local i64) (local.set 0 (i64.const 0)))
+  (func (export "type-local-f32") (local f32) (local.set 0 (f32.const 0)))
+  (func (export "type-local-f64") (local f64) (local.set 0 (f64.const 0)))
+
+  (func (export "type-param-i32") (param i32) (local.set 0 (i32.const 10)))
+  (func (export "type-param-i64") (param i64) (local.set 0 (i64.const 11)))
+  (func (export "type-param-f32") (param f32) (local.set 0 (f32.const 11.1)))
+  (func (export "type-param-f64") (param f64) (local.set 0 (f64.const 12.2)))
+
+  (func (export "type-mixed") (param i64 f32 f64 i32 i32) (local f32 i64 i64 f64)
+    (local.set 0 (i64.const 0))
+    (local.set 1 (f32.const 0))
+    (local.set 2 (f64.const 0))
+    (local.set 3 (i32.const 0))
+    (local.set 4 (i32.const 0))
+    (local.set 5 (f32.const 0))
+    (local.set 6 (i64.const 0))
+    (local.set 7 (i64.const 0))
+    (local.set 8 (f64.const 0))
+  )
+
+  ;; As parameter of control constructs and instructions
+
+  (func (export "as-block-value") (param i32)
+    (block (local.set 0 (i32.const 1)))
+  )
+  (func (export "as-loop-value") (param i32)
+    (loop (local.set 0 (i32.const 3)))
+  )
+
+  (func (export "as-br-value") (param i32)
+    (block (br 0 (local.set 0 (i32.const 9))))
+  )
+  (func (export "as-br_if-value") (param i32)
+    (block
+      (br_if 0 (local.set 0 (i32.const 8)) (i32.const 1))
+    )
+  )
+  (func (export "as-br_if-value-cond") (param i32)
+    (block
+      (br_if 0 (i32.const 6) (local.set 0 (i32.const 9)))
+    )
+  )
+  (func (export "as-br_table-value") (param i32)
+    (block
+      (br_table 0 (local.set 0 (i32.const 10)) (i32.const 1))
+    )
+  )
+
+  (func (export "as-return-value") (param i32)
+    (return (local.set 0 (i32.const 7)))
+  )
+
+  (func (export "as-if-then") (param i32)
+    (if (local.get 0) (then (local.set 0 (i32.const 3))))
+  )
+  (func (export "as-if-else") (param i32)
+    (if (local.get 0) (then) (else (local.set 0 (i32.const 1))))
+  )
+)
+
+(assert_return (invoke "type-local-i32"))
+(assert_return (invoke "type-local-i64"))
+(assert_return (invoke "type-local-f32"))
+(assert_return (invoke "type-local-f64"))
+
+(assert_return (invoke "type-param-i32" (i32.const 2)))
+(assert_return (invoke "type-param-i64" (i64.const 3)))
+(assert_return (invoke "type-param-f32" (f32.const 4.4)))
+(assert_return (invoke "type-param-f64" (f64.const 5.5)))
+
+(assert_return (invoke "as-block-value" (i32.const 0)))
+(assert_return (invoke "as-loop-value" (i32.const 0)))
+
+(assert_return (invoke "as-br-value" (i32.const 0)))
+(assert_return (invoke "as-br_if-value" (i32.const 0)))
+(assert_return (invoke "as-br_if-value-cond" (i32.const 0)))
+(assert_return (invoke "as-br_table-value" (i32.const 0)))
+
+(assert_return (invoke "as-return-value" (i32.const 0)))
+
+(assert_return (invoke "as-if-then" (i32.const 1)))
+(assert_return (invoke "as-if-else" (i32.const 0)))
+
+(assert_return
+  (invoke "type-mixed"
+    (i64.const 1) (f32.const 2.2) (f64.const 3.3) (i32.const 4) (i32.const 5)
+  )
+)
+
+