Repository: bytecodealliance/wasmtime
Pull request #8313 (2024-04-08 07:53:43)

NaN-canonicalization without branching on x64
Modify the cranelift pass that performs NaN-canonicalization to avoid branches on x64. The current implementation uses two branches:
```
       8: be 00 00 c0 7f               	mov	esi, 0x7fc00000
       d: c5 f9 6e de                  	vmovd	xmm3, esi
      11: 0f 2e c0                     	ucomiss	xmm0, xmm0
      14: 0f 8b 04 00 00 00            	jnp	0x1e <wasm[0]::function[0]+0x1e>
      1a: f2 0f 10 c3                  	movsd	xmm0, xmm3              # xmm0 = xmm3[0],xmm0[1]
      1e: 0f 84 04 00 00 00            	je	0x28 <wasm[0]::function[0]+0x28>
      24: f2 0f 10 c3                  	movsd	xmm0, xmm3              # xmm0 = xmm3[0],xmm0[1]
```
With these changes, NaN-canonicalization becomes:
```
       8: c5 e8 c2 da 03               	vcmpunordps	xmm3, xmm2, xmm2
       d: be 00 00 c0 7f               	mov	esi, 0x7fc00000
      12: c5 f9 6e e6                  	vmovd	xmm4, esi
      16: c4 e3 69 4c c4 30            	vpblendvb	xmm0, xmm2, xmm4, xmm3
```

Running both versions against an small image classification inference benchmark [here](https://github.com/adambratschikaye/image-classification-wasmtime-bench) resulted in a ~50% improvement:
```
image_classification/opt-level=0
                        time:   [728.16 ms 730.00 ms 732.05 ms]
                        change: [-44.476% -44.251% -44.029%] (p = 0.00 < 0.05)
                        Performance has improved.

image_classification/opt-level=1
                        time:   [593.90 ms 595.51 ms 597.34 ms]
                        change: [-51.561% -51.396% -51.211%] (p = 0.00 < 0.05)
                        Performance has improved.
```

As a side note, I didn't notice any sightglass benchmark that was performing mainly float arithmetic to test against. I'd be happy to add this image classification case if there's interest. 

<!--
Please make sure you include the following information:

- If this work has been discussed elsewhere, please include a link to that
  conversation. If it was discussed in an issue, just mention "issue #...".

- Explain why this change is needed. If the details are in an issue already,
  this can be brief.

Our development process is documented in the Wasmtime book:
https://docs.wasmtime.dev/contributing-development-process.html

Please ensure all communication follows the code of conduct:
https://github.com/bytecodealliance/wasmtime/blob/main/CODE_OF_CONDUCT.md
-->


adambratschikaye: Also is there a way to enable NaN-canonicalization in a `clif` test to add a test for this?
afonso360: You should be able to add something along these lines, to test with nan canonicalization enabled:
```
test {run,compile,etc...}
set enable_nan_canonicalization=true
target x86_64
```


Patch:
diff --git a/cranelift/codegen/src/bitset.rs b/cranelift/codegen/src/bitset.rs
index 7d81b3221..70c86d722 100644
--- a/cranelift/codegen/src/bitset.rs
+++ b/cranelift/codegen/src/bitset.rs
@@ -10,13 +10,35 @@ use core::mem::size_of;
 use core::ops::{Add, BitOr, Shl, Sub};
 
 /// A small bitset built on a single primitive integer type
-#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Copy, Default, PartialEq, Eq)]
 #[cfg_attr(
     feature = "enable-serde",
     derive(serde_derive::Serialize, serde_derive::Deserialize)
 )]
 pub struct BitSet<T>(pub T);
 
+impl<T> std::fmt::Debug for BitSet<T>
+where
+    T: Into<u32>
+        + From<u8>
+        + BitOr<T, Output = T>
+        + Shl<u8, Output = T>
+        + Sub<T, Output = T>
+        + Add<T, Output = T>
+        + PartialEq
+        + Copy,
+{
+    fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+        let mut s = f.debug_struct(std::any::type_name::<Self>());
+        for i in 0..Self::bits() {
+            use std::string::ToString;
+            let i = u32::try_from(i).unwrap();
+            s.field(&i.to_string(), &self.contains(i));
+        }
+        s.finish()
+    }
+}
+
 impl<T> BitSet<T>
 where
     T: Into<u32>
diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs
index c5efafaee..93e955c9a 100644
--- a/cranelift/codegen/src/context.rs
+++ b/cranelift/codegen/src/context.rs
@@ -31,6 +31,7 @@ use crate::{timing, CompileError};
 use alloc::string::String;
 use alloc::vec::Vec;
 use cranelift_control::ControlPlane;
+use target_lexicon::Architecture;
 
 #[cfg(feature = "souper-harvest")]
 use crate::souper_harvest::do_souper_harvest;
@@ -282,7 +283,15 @@ impl Context {
 
     /// Perform NaN canonicalizing rewrites on the function.
     pub fn canonicalize_nans(&mut self, isa: &dyn TargetIsa) -> CodegenResult<()> {
-        do_nan_canonicalization(&mut self.func);
+        // Currently only RiscV64 is the only arch that may not have vector support.
+        let has_vector_support = match isa.triple().architecture {
+            Architecture::Riscv64(_) => match isa.isa_flags().iter().find(|f| f.name == "has_v") {
+                Some(value) => value.as_bool().unwrap_or(false),
+                None => false,
+            },
+            _ => true,
+        };
+        do_nan_canonicalization(&mut self.func, has_vector_support);
         self.verify_if(isa)
     }
 
diff --git a/cranelift/codegen/src/inst_predicates.rs b/cranelift/codegen/src/inst_predicates.rs
index 707a01c1a..c0e7ed665 100644
--- a/cranelift/codegen/src/inst_predicates.rs
+++ b/cranelift/codegen/src/inst_predicates.rs
@@ -44,6 +44,17 @@ pub fn has_side_effect(func: &Function, inst: Inst) -> bool {
     trivially_has_side_effects(opcode) || is_load_with_defined_trapping(opcode, data)
 }
 
+/// Is the given instruction a bitcast to or from a reference type (e.g. `r64`)?
+pub fn is_bitcast_from_ref(func: &Function, inst: Inst) -> bool {
+    let op = func.dfg.insts[inst].opcode();
+    if op != ir::Opcode::Bitcast {
+        return false;
+    }
+
+    let arg = func.dfg.inst_args(inst)[0];
+    func.dfg.value_type(arg).is_ref()
+}
+
 /// Does the given instruction behave as a "pure" node with respect to
 /// aegraph semantics?
 ///
@@ -58,6 +69,7 @@ pub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {
         } => flags.readonly() && flags.notrap(),
         _ => false,
     };
+
     // Multi-value results do not play nicely with much of the egraph
     // infrastructure. They are in practice used only for multi-return
     // calls and some other odd instructions (e.g. uadd_overflow) which,
@@ -70,7 +82,11 @@ pub fn is_pure_for_egraph(func: &Function, inst: Inst) -> bool {
 
     let op = func.dfg.insts[inst].opcode();
 
-    has_one_result && (is_readonly_load || (!op.can_load() && !trivially_has_side_effects(op)))
+    has_one_result
+        && (is_readonly_load || (!op.can_load() && !trivially_has_side_effects(op)))
+        // Cannot optimize ref-y bitcasts, as that can interact badly with
+        // safepoints and stack maps.
+        && !is_bitcast_from_ref(func, inst)
 }
 
 /// Can the given instruction be merged into another copy of itself?
@@ -90,6 +106,9 @@ pub fn is_mergeable_for_egraph(func: &Function, inst: Inst) -> bool {
         && !op.can_store()
         // Can only have idempotent side-effects.
         && (!has_side_effect(func, inst) || op.side_effects_idempotent())
+        // Cannot optimize ref-y bitcasts, as that can interact badly with
+        // safepoints and stack maps.
+        && !is_bitcast_from_ref(func, inst)
 }
 
 /// Does the given instruction have any side-effect as per [has_side_effect], or else is a load,
diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs
index 94b5897b6..8139ac81f 100644
--- a/cranelift/codegen/src/isa/aarch64/abi.rs
+++ b/cranelift/codegen/src/isa/aarch64/abi.rs
@@ -36,9 +36,9 @@ impl Into<AMode> for StackAMode {
     fn into(self) -> AMode {
         match self {
             // Argument area begins after saved frame pointer + return address.
-            StackAMode::ArgOffset(off, _ty) => AMode::FPOffset { off: off + 16 },
-            StackAMode::NominalSPOffset(off, _ty) => AMode::NominalSPOffset { off },
-            StackAMode::SPOffset(off, _ty) => AMode::SPOffset { off },
+            StackAMode::IncomingArg(off) => AMode::FPOffset { off: off + 16 },
+            StackAMode::Slot(off) => AMode::NominalSPOffset { off },
+            StackAMode::OutgoingArg(off) => AMode::SPOffset { off },
         }
     }
 }
@@ -457,7 +457,7 @@ impl ABIMachineSpec for AArch64MachineDeps {
         insts
     }
 
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, _ty: Type) -> Inst {
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Inst {
         // FIXME: Do something different for dynamic types?
         let mem = mem.into();
         Inst::LoadAddr { rd: into_reg, mem }
diff --git a/cranelift/codegen/src/isa/riscv64/abi.rs b/cranelift/codegen/src/isa/riscv64/abi.rs
index 8f99caa69..dca2cda68 100644
--- a/cranelift/codegen/src/isa/riscv64/abi.rs
+++ b/cranelift/codegen/src/isa/riscv64/abi.rs
@@ -293,7 +293,7 @@ impl ABIMachineSpec for Riscv64MachineDeps {
         insts
     }
 
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, _ty: Type) -> Inst {
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Inst {
         Inst::LoadAddr {
             rd: into_reg,
             mem: mem.into(),
diff --git a/cranelift/codegen/src/isa/riscv64/inst/args.rs b/cranelift/codegen/src/isa/riscv64/inst/args.rs
index 9a9fe4d8f..4f8c7beed 100644
--- a/cranelift/codegen/src/isa/riscv64/inst/args.rs
+++ b/cranelift/codegen/src/isa/riscv64/inst/args.rs
@@ -200,9 +200,9 @@ impl Into<AMode> for StackAMode {
     fn into(self) -> AMode {
         match self {
             // Argument area begins after saved lr + fp.
-            StackAMode::ArgOffset(offset, _ty) => AMode::FPOffset(offset + 16),
-            StackAMode::SPOffset(offset, _ty) => AMode::SPOffset(offset),
-            StackAMode::NominalSPOffset(offset, _ty) => AMode::NominalSPOffset(offset),
+            StackAMode::IncomingArg(offset) => AMode::FPOffset(offset + 16),
+            StackAMode::OutgoingArg(offset) => AMode::SPOffset(offset),
+            StackAMode::Slot(offset) => AMode::NominalSPOffset(offset),
         }
     }
 }
diff --git a/cranelift/codegen/src/isa/s390x/abi.rs b/cranelift/codegen/src/isa/s390x/abi.rs
index b5fb077bf..55ebf6b2f 100644
--- a/cranelift/codegen/src/isa/s390x/abi.rs
+++ b/cranelift/codegen/src/isa/s390x/abi.rs
@@ -192,9 +192,9 @@ impl Into<MemArg> for StackAMode {
     fn into(self) -> MemArg {
         match self {
             // Argument area always begins at the initial SP.
-            StackAMode::ArgOffset(off, _ty) => MemArg::InitialSPOffset { off },
-            StackAMode::NominalSPOffset(off, _ty) => MemArg::NominalSPOffset { off },
-            StackAMode::SPOffset(off, _ty) => {
+            StackAMode::IncomingArg(off) => MemArg::InitialSPOffset { off },
+            StackAMode::Slot(off) => MemArg::NominalSPOffset { off },
+            StackAMode::OutgoingArg(off) => {
                 MemArg::reg_plus_off(stack_reg(), off, MemFlags::trusted())
             }
         }
@@ -495,7 +495,7 @@ impl ABIMachineSpec for S390xMachineDeps {
         insts
     }
 
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, _ty: Type) -> Inst {
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Inst {
         let mem = mem.into();
         Inst::LoadAddr { rd: into_reg, mem }
     }
diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs
index b3eaea14c..a4c36a523 100644
--- a/cranelift/codegen/src/isa/x64/abi.rs
+++ b/cranelift/codegen/src/isa/x64/abi.rs
@@ -478,7 +478,7 @@ impl ABIMachineSpec for X64ABIMachineSpec {
         ]
     }
 
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, _ty: Type) -> Self::I {
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Self::I {
         let mem: SyntheticAmode = mem.into();
         Inst::lea(mem, into_reg)
     }
@@ -1018,27 +1018,26 @@ impl From<StackAMode> for SyntheticAmode {
         // We enforce a 128 MB stack-frame size limit above, so these
         // `expect()`s should never fail.
         match amode {
-            StackAMode::ArgOffset(off, _ty) => {
-                let off =
-                    i32::try_from(off + 16) // frame pointer + return address
-                        .expect(
-                            "Offset in ArgOffset is greater than 2GB; should hit impl limit first",
-                        );
+            StackAMode::IncomingArg(off) => {
+                let off = i32::try_from(off + 16) // frame pointer + return address
+                    .expect(
+                        "Offset in IncomingArg is greater than 2GB; should hit impl limit first",
+                    );
                 SyntheticAmode::Real(Amode::ImmReg {
                     simm32: off,
                     base: regs::rbp(),
                     flags: MemFlags::trusted(),
                 })
             }
-            StackAMode::NominalSPOffset(off, _ty) => {
-                let off = i32::try_from(off).expect(
-                    "Offset in NominalSPOffset is greater than 2GB; should hit impl limit first",
-                );
+            StackAMode::Slot(off) => {
+                let off = i32::try_from(off)
+                    .expect("Offset in Slot is greater than 2GB; should hit impl limit first");
                 SyntheticAmode::nominal_sp_offset(off)
             }
-            StackAMode::SPOffset(off, _ty) => {
-                let off = i32::try_from(off)
-                    .expect("Offset in SPOffset is greater than 2GB; should hit impl limit first");
+            StackAMode::OutgoingArg(off) => {
+                let off = i32::try_from(off).expect(
+                    "Offset in OutgoingArg is greater than 2GB; should hit impl limit first",
+                );
                 SyntheticAmode::Real(Amode::ImmReg {
                     simm32: off,
                     base: regs::rsp(),
diff --git a/cranelift/codegen/src/isa/x64/lower.isle b/cranelift/codegen/src/isa/x64/lower.isle
index 64f738053..e180d35bb 100644
--- a/cranelift/codegen/src/isa/x64/lower.isle
+++ b/cranelift/codegen/src/isa/x64/lower.isle
@@ -1428,6 +1428,7 @@
 (decl pure partial all_ones_or_all_zeros (Value) bool)
 (rule (all_ones_or_all_zeros (and (icmp _ _ _) (value_type (multi_lane _ _)))) $true)
 (rule (all_ones_or_all_zeros (and (fcmp _ _ _) (value_type (multi_lane _ _)))) $true)
+(rule (all_ones_or_all_zeros (and (bitcast _ (fcmp _ _ _)) (value_type (multi_lane _ _)))) $true)
 (rule (all_ones_or_all_zeros (vconst (vconst_all_ones_or_all_zeros))) $true)
 
 (decl pure vconst_all_ones_or_all_zeros () Constant)
diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs
index 3f195baaa..bb65431a2 100644
--- a/cranelift/codegen/src/machinst/abi.rs
+++ b/cranelift/codegen/src/machinst/abi.rs
@@ -281,15 +281,12 @@ pub enum ArgsOrRets {
 /// appropriate addressing mode.
 #[derive(Clone, Copy, Debug)]
 pub enum StackAMode {
-    /// Offset into the current frame's argument area, possibly making use of a
-    /// specific type for a scaled indexing operation.
-    ArgOffset(i64, ir::Type),
-    /// Offset from the nominal stack pointer, possibly making use of a specific
-    /// type for a scaled indexing operation.
-    NominalSPOffset(i64, ir::Type),
-    /// Offset from the real stack pointer, possibly making use of a specific
-    /// type for a scaled indexing operation.
-    SPOffset(i64, ir::Type),
+    /// Offset into the current frame's argument area.
+    IncomingArg(i64),
+    /// Offset within the stack slots in the current frame.
+    Slot(i64),
+    /// Offset into the callee frame's argument area.
+    OutgoingArg(i64),
 }
 
 /// Trait implemented by machine-specific backend to represent ISA flags.
@@ -447,7 +444,7 @@ pub trait ABIMachineSpec {
 
     /// Generate an instruction to compute an address of a stack slot (FP- or
     /// SP-based offset).
-    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>, ty: Type) -> Self::I;
+    fn gen_get_stack_addr(mem: StackAMode, into_reg: Writable<Reg>) -> Self::I;
 
     /// Get a fixed register to use to compute a stack limit. This is needed for
     /// certain sequences generated after the register allocator has already
@@ -1475,7 +1472,7 @@ impl<M: ABIMachineSpec> Callee<M> {
                             ty
                         };
                     insts.push(M::gen_load_stack(
-                        StackAMode::ArgOffset(offset, ty),
+                        StackAMode::IncomingArg(offset),
                         *into_reg,
                         ty,
                     ));
@@ -1500,9 +1497,8 @@ impl<M: ABIMachineSpec> Callee<M> {
                 } else {
                     // Buffer address is implicitly defined by the ABI.
                     insts.push(M::gen_get_stack_addr(
-                        StackAMode::ArgOffset(offset, I8),
+                        StackAMode::IncomingArg(offset),
                         into_reg,
-                        I8,
                     ));
                 }
             }
@@ -1523,7 +1519,7 @@ impl<M: ABIMachineSpec> Callee<M> {
                         // This was allocated in the `init` routine.
                         let addr_reg = self.arg_temp_reg[idx].unwrap();
                         insts.push(M::gen_load_stack(
-                            StackAMode::ArgOffset(offset, ty),
+                            StackAMode::IncomingArg(offset),
                             addr_reg,
                             ty,
                         ));
@@ -1694,17 +1690,13 @@ impl<M: ABIMachineSpec> Callee<M> {
         // [MemArg::NominalSPOffset] for more details on nominal SP tracking).
         let stack_off = self.sized_stackslots[slot] as i64;
         let sp_off: i64 = stack_off + (offset as i64);
-        M::gen_get_stack_addr(StackAMode::NominalSPOffset(sp_off, I8), into_reg, I8)
+        M::gen_get_stack_addr(StackAMode::Slot(sp_off), into_reg)
     }
 
     /// Produce an instruction that computes a dynamic stackslot address.
     pub fn dynamic_stackslot_addr(&self, slot: DynamicStackSlot, into_reg: Writable<Reg>) -> M::I {
         let stack_off = self.dynamic_stackslots[slot] as i64;
-        M::gen_get_stack_addr(
-            StackAMode::NominalSPOffset(stack_off, I64X2XN),
-            into_reg,
-            I64X2XN,
-        )
+        M::gen_get_stack_addr(StackAMode::Slot(stack_off), into_reg)
     }
 
     /// Get an `args` pseudo-inst, if any, that should appear at the
@@ -1977,7 +1969,7 @@ impl<M: ABIMachineSpec> Callee<M> {
         let sp_off = self.get_spillslot_offset(to_slot);
         trace!("gen_spill: {from_reg:?} into slot {to_slot:?} at offset {sp_off}");
 
-        let from = StackAMode::NominalSPOffset(sp_off, ty);
+        let from = StackAMode::Slot(sp_off);
         <M>::gen_store_stack(from, Reg::from(from_reg), ty)
     }
 
@@ -1989,7 +1981,7 @@ impl<M: ABIMachineSpec> Callee<M> {
         let sp_off = self.get_spillslot_offset(from_slot);
         trace!("gen_reload: {to_reg:?} from slot {from_slot:?} at offset {sp_off}");
 
-        let from = StackAMode::NominalSPOffset(sp_off, ty);
+        let from = StackAMode::Slot(sp_off);
         <M>::gen_load_stack(from, to_reg.map(Reg::from), ty)
     }
 }
@@ -2242,9 +2234,8 @@ impl<M: ABIMachineSpec> CallSite<M> {
                 let src_ptr = from_regs.only_reg().unwrap();
                 let dst_ptr = ctx.alloc_tmp(M::word_type()).only_reg().unwrap();
                 ctx.emit(M::gen_get_stack_addr(
-                    StackAMode::SPOffset(offset, I8),
+                    StackAMode::OutgoingArg(offset),
                     dst_ptr,
-                    I8,
                 ));
                 // Emit a memcpy from `src_ptr` to `dst_ptr` of `size` bytes.
                 // N.B.: because we process StructArg params *first*, this is
@@ -2283,9 +2274,9 @@ impl<M: ABIMachineSpec> CallSite<M> {
                             "tail calls require frame pointers to be enabled"
                         );
 
-                        StackAMode::ArgOffset(offset, ty)
+                        StackAMode::IncomingArg(offset)
                     } else {
-                        StackAMode::SPOffset(offset, ty)
+                        StackAMode::OutgoingArg(offset)
                     };
                     ctx.emit(M::gen_store_stack(amode, vreg, ty))
                 }
@@ -2391,9 +2382,9 @@ impl<M: ABIMachineSpec> CallSite<M> {
             } => {
                 assert_eq!(from_regs.len(), 1);
                 let vreg = from_regs.regs()[0];
-                let amode = StackAMode::SPOffset(offset, ty);
+                let amode = StackAMode::OutgoingArg(offset);
                 let tmp = ctx.alloc_tmp(M::word_type()).only_reg().unwrap();
-                ctx.emit(M::gen_get_stack_addr(amode, tmp, ty));
+                ctx.emit(M::gen_get_stack_addr(amode, tmp));
                 let tmp = tmp.to_reg();
                 ctx.emit(M::gen_store_base_offset(tmp, 0, vreg, ty));
                 let loc = match pointer {
@@ -2504,7 +2495,7 @@ impl<M: ABIMachineSpec> CallSite<M> {
                                 sig_data.sized_stack_arg_space()
                             };
                             insts.push(M::gen_load_stack(
-                                StackAMode::SPOffset(offset + ret_area_base, ty),
+                                StackAMode::OutgoingArg(offset + ret_area_base),
                                 *into_reg,
                                 ty,
                             ));
@@ -2541,9 +2532,8 @@ impl<M: ABIMachineSpec> CallSite<M> {
             let rd = ctx.alloc_tmp(word_type).only_reg().unwrap();
             let ret_area_base = ctx.sigs()[self.sig].sized_stack_arg_space();
             ctx.emit(M::gen_get_stack_addr(
-                StackAMode::SPOffset(ret_area_base, I8),
+                StackAMode::OutgoingArg(ret_area_base),
                 rd,
-                I8,
             ));
             let moves = self.gen_arg(ctx, i.into(), ValueRegs::one(rd.to_reg()));
             self.emit_arg_moves(ctx, moves);
diff --git a/cranelift/codegen/src/machinst/buffer.rs b/cranelift/codegen/src/machinst/buffer.rs
index 8abe64a68..a18adea65 100644
--- a/cranelift/codegen/src/machinst/buffer.rs
+++ b/cranelift/codegen/src/machinst/buffer.rs
@@ -1675,7 +1675,7 @@ impl<I: VCodeInst> MachBuffer<I> {
                 (start_offset, end_offset)
             }
         };
-        trace!("Adding stack map for offsets {start:#x}..{end:#x}");
+        trace!("Adding stack map for offsets {start:#x}..{end:#x}: {stack_map:?}");
         self.stack_maps.push(MachStackMap {
             offset: start,
             offset_end: end,
diff --git a/cranelift/codegen/src/nan_canonicalization.rs b/cranelift/codegen/src/nan_canonicalization.rs
index c1bc16f77..5be41857a 100644
--- a/cranelift/codegen/src/nan_canonicalization.rs
+++ b/cranelift/codegen/src/nan_canonicalization.rs
@@ -5,7 +5,7 @@
 use crate::cursor::{Cursor, FuncCursor};
 use crate::ir::condcodes::FloatCC;
 use crate::ir::immediates::{Ieee32, Ieee64};
-use crate::ir::types;
+use crate::ir::types::{self};
 use crate::ir::{Function, Inst, InstBuilder, InstructionData, Opcode, Value};
 use crate::opts::MemFlags;
 use crate::timing;
@@ -15,13 +15,13 @@ static CANON_32BIT_NAN: u32 = 0b01111111110000000000000000000000;
 static CANON_64BIT_NAN: u64 = 0b0111111111111000000000000000000000000000000000000000000000000000;
 
 /// Perform the NaN canonicalization pass.
-pub fn do_nan_canonicalization(func: &mut Function) {
+pub fn do_nan_canonicalization(func: &mut Function, has_vector_support: bool) {
     let _tt = timing::canonicalize_nans();
     let mut pos = FuncCursor::new(func);
     while let Some(_block) = pos.next_block() {
         while let Some(inst) = pos.next_inst() {
             if is_fp_arith(&mut pos, inst) {
-                add_nan_canon_seq(&mut pos, inst);
+                add_nan_canon_seq(&mut pos, inst, has_vector_support);
             }
         }
     }
@@ -57,7 +57,7 @@ fn is_fp_arith(pos: &mut FuncCursor, inst: Inst) -> bool {
 }
 
 /// Append a sequence of canonicalizing instructions after the given instruction.
-fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst) {
+fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst, has_vector_support: bool) {
     // Select the instruction result, result type. Replace the instruction
     // result and step forward before inserting the canonicalization sequence.
     let val = pos.func.dfg.first_result(inst);
@@ -65,16 +65,28 @@ fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst) {
     let new_res = pos.func.dfg.replace_result(val, val_type);
     let _next_inst = pos.next_inst().expect("block missing terminator!");
 
-    // Insert a comparison instruction, to check if `inst_res` is NaN. Select
-    // the canonical NaN value if `val` is NaN, assign the result to `inst`.
-    let is_nan = pos.ins().fcmp(FloatCC::NotEqual, new_res, new_res);
+    // Insert a comparison instruction, to check if `inst_res` is NaN (comparing
+    // against NaN is always unordered). Select the canonical NaN value if `val`
+    // is NaN, assign the result to `inst`.
+    let comparison = FloatCC::Unordered;
 
+    let vectorized_scalar_select = |pos: &mut FuncCursor, canon_nan: Value, ty: types::Type| {
+        let canon_nan = pos.ins().scalar_to_vector(ty, canon_nan);
+        let new_res = pos.ins().scalar_to_vector(ty, new_res);
+        let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
+        let is_nan = pos.ins().bitcast(ty, MemFlags::new(), is_nan);
+        let simd_result = pos.ins().bitselect(is_nan, canon_nan, new_res);
+        pos.ins().with_result(val).extractlane(simd_result, 0);
+    };
     let scalar_select = |pos: &mut FuncCursor, canon_nan: Value| {
+        let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
         pos.ins()
             .with_result(val)
             .select(is_nan, canon_nan, new_res);
     };
+
     let vector_select = |pos: &mut FuncCursor, canon_nan: Value| {
+        let is_nan = pos.ins().fcmp(comparison, new_res, new_res);
         let is_nan = pos.ins().bitcast(val_type, MemFlags::new(), is_nan);
         pos.ins()
             .with_result(val)
@@ -84,11 +96,19 @@ fn add_nan_canon_seq(pos: &mut FuncCursor, inst: Inst) {
     match val_type {
         types::F32 => {
             let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN));
-            scalar_select(pos, canon_nan);
+            if has_vector_support {
+                vectorized_scalar_select(pos, canon_nan, types::F32X4);
+            } else {
+                scalar_select(pos, canon_nan);
+            }
         }
         types::F64 => {
             let canon_nan = pos.ins().f64const(Ieee64::with_bits(CANON_64BIT_NAN));
-            scalar_select(pos, canon_nan);
+            if has_vector_support {
+                vectorized_scalar_select(pos, canon_nan, types::F64X2);
+            } else {
+                scalar_select(pos, canon_nan);
+            }
         }
         types::F32X4 => {
             let canon_nan = pos.ins().f32const(Ieee32::with_bits(CANON_32BIT_NAN));
diff --git a/crates/runtime/src/gc/enabled/drc.rs b/crates/runtime/src/gc/enabled/drc.rs
index 6987984da..56bba9838 100644
--- a/crates/runtime/src/gc/enabled/drc.rs
+++ b/crates/runtime/src/gc/enabled/drc.rs
@@ -160,6 +160,12 @@ impl DrcHeap {
 
         let drc_ref = drc_ref(gc_ref);
         let header = self.index_mut(&drc_ref);
+        debug_assert_ne!(
+            *header.ref_count.get_mut(),
+            0,
+            "{:#p} is supposedly live; should have nonzero ref count",
+            *gc_ref
+        );
         *header.ref_count.get_mut() += 1;
         log::trace!(
             "increment {:#p} ref count -> {}",
@@ -179,6 +185,12 @@ impl DrcHeap {
 
         let drc_ref = drc_ref(gc_ref);
         let header = self.index_mut(drc_ref);
+        debug_assert_ne!(
+            *header.ref_count.get_mut(),
+            0,
+            "{:#p} is supposedly live; should have nonzero ref count",
+            *gc_ref
+        );
         *header.ref_count.get_mut() -= 1;
         log::trace!(
             "decrement {:#p} ref count -> {}",
@@ -261,6 +273,13 @@ impl DrcHeap {
                 continue;
             }
 
+            debug_assert_ne!(
+                *self.index_mut(drc_ref(&gc_ref)).ref_count.get_mut(),
+                0,
+                "{gc_ref:#p} is on the Wasm stack and therefore should be held \
+                 by the activations table; should have nonzero ref count",
+            );
+
             log::trace!("Found GC reference on the stack: {:#p}", gc_ref);
             let is_new = self
                 .activations_table
@@ -272,26 +291,76 @@ impl DrcHeap {
         }
     }
 
-    /// Sweep the bump allocation table after we've discovered our precise stack
-    /// roots.
-    fn sweep(&mut self, host_data_table: &mut ExternRefHostDataTable) {
-        // Sweep our bump chunk.
+    fn iter_bump_chunk(&mut self) -> impl Iterator<Item = VMGcRef> + '_ {
         let num_filled = self.activations_table.num_filled_in_bump_chunk();
+        self.activations_table
+            .alloc
+            .chunk
+            .iter_mut()
+            .take(num_filled)
+            .map(|slot| {
+                let r64 = *slot.get_mut();
+                VMGcRef::from_r64(r64)
+                    .expect("valid r64")
+                    .expect("non-null")
+            })
+    }
+
+    #[inline(never)]
+    #[cold]
+    fn log_gc_ref_set(prefix: &str, items: impl Iterator<Item = VMGcRef>) {
+        assert!(log::log_enabled!(log::Level::Trace));
+        let mut set = "{".to_string();
+        let mut any = false;
+        for gc_ref in items {
+            any = true;
+            set += &format!("\n  {gc_ref:#p},");
+        }
+        if any {
+            set.push('\n');
+        }
+        set.push('}');
+        log::trace!("{prefix}: {set}");
+    }
+
+    fn drain_bump_chunk(&mut self, mut f: impl FnMut(&mut Self, VMGcRef)) {
+        let num_filled = self.activations_table.num_filled_in_bump_chunk();
+
+        // Temporarily take the allocation out of `self` to avoid conflicting
+        // borrows.
         let mut alloc = mem::take(&mut self.activations_table.alloc);
         for slot in alloc.chunk.iter_mut().take(num_filled) {
             let r64 = mem::take(slot.get_mut());
             let gc_ref = VMGcRef::from_r64(r64)
                 .expect("valid r64")
                 .expect("non-null");
-            self.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref);
-
+            f(self, gc_ref);
             *slot.get_mut() = 0;
         }
+
         debug_assert!(
-            alloc.chunk.iter().all(|slot| unsafe { *slot.get() == 0 }),
-            "after sweeping the bump chunk, all slots should be empty"
+            alloc.chunk.iter_mut().all(|slot| *slot.get_mut() == 0),
+            "after sweeping the bump chunk, all slots should be empty",
         );
 
+        debug_assert!(self.activations_table.alloc.chunk.is_empty());
+        self.activations_table.alloc = alloc;
+    }
+
+    /// Sweep the bump allocation table after we've discovered our precise stack
+    /// roots.
+    fn sweep(&mut self, host_data_table: &mut ExternRefHostDataTable) {
+        if log::log_enabled!(log::Level::Trace) {
+            Self::log_gc_ref_set("bump chunk before sweeping", self.iter_bump_chunk());
+        }
+
+        // Sweep our bump chunk.
+        log::trace!("Begin sweeping bump chunk");
+        self.drain_bump_chunk(|heap, gc_ref| {
+            heap.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref);
+        });
+        log::trace!("Done sweeping bump chunk");
+
         if self.activations_table.alloc.chunk.is_empty() {
             // If this is the first collection, then the bump chunk is empty
             // since we lazily allocate it. Force that lazy allocation now so we
@@ -302,6 +371,16 @@ impl DrcHeap {
             self.activations_table.alloc.reset();
         }
 
+        if log::log_enabled!(log::Level::Trace) {
+            Self::log_gc_ref_set(
+                "hash set before sweeping",
+                self.activations_table
+                    .over_approximated_stack_roots
+                    .iter()
+                    .map(|r| r.unchecked_copy()),
+            );
+        }
+
         // The current `precise_stack_roots` becomes our new over-appoximated
         // set for the next GC cycle.
         mem::swap(
@@ -315,14 +394,26 @@ impl DrcHeap {
         // Note that this may run arbitrary code as we run gc_ref
         // destructors. Because of our `&mut` borrow above on this table,
         // though, we're guaranteed that nothing will touch this table.
+        log::trace!("Begin sweeping hash set");
         let mut precise_stack_roots = mem::take(&mut self.activations_table.precise_stack_roots);
         for gc_ref in precise_stack_roots.drain() {
             self.dec_ref_and_maybe_dealloc(host_data_table, &gc_ref);
         }
+        log::trace!("Done sweeping hash set");
 
         // Make sure to replace the `precise_stack_roots` so that we reuse any
         // allocated capacity.
         self.activations_table.precise_stack_roots = precise_stack_roots;
+
+        if log::log_enabled!(log::Level::Trace) {
+            Self::log_gc_ref_set(
+                "hash set after sweeping",
+                self.activations_table
+                    .over_approximated_stack_roots
+                    .iter()
+                    .map(|r| r.unchecked_copy()),
+            );
+        }
     }
 }
 
diff --git a/crates/runtime/src/gc/gc_runtime.rs b/crates/runtime/src/gc/gc_runtime.rs
index e3cbff94f..1d2741dc5 100644
--- a/crates/runtime/src/gc/gc_runtime.rs
+++ b/crates/runtime/src/gc/gc_runtime.rs
@@ -311,19 +311,22 @@ enum RawGcRoot {
 
 impl GcRootsList {
     /// Add a GC root that is inside a Wasm stack frame to this list.
-    //
-    // TODO: we shouldn't need to differentiate between on-stack GC roots and
-    // others, and both should just be `*mut VMGcRef`, but because Cranelift
-    // doesn't support `r32` on 64-bit platforms yet, we have to use `r64` for
-    // roots on the Wasm stack instead.
     #[inline]
-    pub fn add_wasm_stack_root(&mut self, ptr_to_root: SendSyncPtr<u64>) {
+    pub unsafe fn add_wasm_stack_root(&mut self, ptr_to_root: SendSyncPtr<u64>) {
+        log::trace!(
+            "Adding Wasm stack root: {:#p}",
+            VMGcRef::from_r64(*ptr_to_root.as_ref()).unwrap().unwrap()
+        );
         self.0.push(RawGcRoot::Stack(ptr_to_root));
     }
 
     /// Add a GC root to this list.
     #[inline]
-    pub fn add_root(&mut self, ptr_to_root: SendSyncPtr<VMGcRef>) {
+    pub unsafe fn add_root(&mut self, ptr_to_root: SendSyncPtr<VMGcRef>) {
+        log::trace!(
+            "Adding non-stack root: {:#p}",
+            ptr_to_root.as_ref().unchecked_copy()
+        );
         self.0.push(RawGcRoot::NonStack(ptr_to_root))
     }
 
diff --git a/crates/runtime/src/gc/host_data.rs b/crates/runtime/src/gc/host_data.rs
index 933935bed..13161b508 100644
--- a/crates/runtime/src/gc/host_data.rs
+++ b/crates/runtime/src/gc/host_data.rs
@@ -39,11 +39,14 @@ impl ExternRefHostDataTable {
     /// Allocate a new `externref` host data value.
     pub fn alloc(&mut self, value: Box<dyn Any + Send + Sync>) -> ExternRefHostDataId {
         let id = self.slab.alloc(value);
-        ExternRefHostDataId(id)
+        let id = ExternRefHostDataId(id);
+        log::trace!("allocated new externref host data: {id:?}");
+        id
     }
 
     /// Deallocate an `externref` host data value.
     pub fn dealloc(&mut self, id: ExternRefHostDataId) -> Box<dyn Any + Send + Sync> {
+        log::trace!("deallocated externref host data: {id:?}");
         self.slab.dealloc(id.0)
     }
 
diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs
index 1095948ba..c2074eeee 100644
--- a/crates/runtime/src/libcalls.rs
+++ b/crates/runtime/src/libcalls.rs
@@ -380,6 +380,7 @@ unsafe fn drop_gc_ref(instance: &mut Instance, gc_ref: *mut u8) {
     let gc_ref = VMGcRef::from_r64(u64::try_from(gc_ref as usize).unwrap())
         .expect("valid r64")
         .expect("non-null VMGcRef");
+    log::trace!("libcalls::drop_gc_ref({gc_ref:?})");
     (*instance.store()).gc_store().drop_gc_ref(gc_ref);
 }
 
diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs
index 4a0ea6019..f138d1a37 100644
--- a/crates/runtime/src/table.rs
+++ b/crates/runtime/src/table.rs
@@ -467,16 +467,16 @@ impl Table {
             TableElement::FuncRef(f) => {
                 self.funcrefs_mut()[start..end].fill(TaggedFuncRef::from(f));
             }
-            TableElement::GcRef(e) => {
+            TableElement::GcRef(r) => {
                 // Clone the init GC reference into each table slot.
                 for slot in &mut self.gc_refs_mut()[start..end] {
-                    gc_store.write_gc_ref(slot, e.as_ref());
+                    gc_store.write_gc_ref(slot, r.as_ref());
                 }
 
                 // Drop the init GC reference, since we aren't holding onto this
                 // reference anymore, only the clones in the table.
-                if let Some(e) = e {
-                    gc_store.drop_gc_ref(e);
+                if let Some(r) = r {
+                    gc_store.drop_gc_ref(r);
                 }
             }
             TableElement::UninitFunc => {
diff --git a/crates/wasmtime/src/runtime/externals/global.rs b/crates/wasmtime/src/runtime/externals/global.rs
index 8276e6b98..dc58268d2 100644
--- a/crates/wasmtime/src/runtime/externals/global.rs
+++ b/crates/wasmtime/src/runtime/externals/global.rs
@@ -213,7 +213,9 @@ impl Global {
             if let Some(gc_ref) = unsafe { (*store[self.0].definition).as_gc_ref() } {
                 let gc_ref = NonNull::from(gc_ref);
                 let gc_ref = SendSyncPtr::new(gc_ref);
-                gc_roots_list.add_root(gc_ref);
+                unsafe {
+                    gc_roots_list.add_root(gc_ref);
+                }
             }
         }
     }
diff --git a/crates/wasmtime/src/runtime/externals/table.rs b/crates/wasmtime/src/runtime/externals/table.rs
index 145723a36..0adc6ad94 100644
--- a/crates/wasmtime/src/runtime/externals/table.rs
+++ b/crates/wasmtime/src/runtime/externals/table.rs
@@ -387,7 +387,9 @@ impl Table {
             if let Some(gc_ref) = gc_ref {
                 let gc_ref = NonNull::from(gc_ref);
                 let gc_ref = SendSyncPtr::new(gc_ref);
-                gc_roots_list.add_root(gc_ref);
+                unsafe {
+                    gc_roots_list.add_root(gc_ref);
+                }
             }
         }
     }
diff --git a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
index 06193a529..293765e0a 100644
--- a/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
+++ b/crates/wasmtime/src/runtime/gc/enabled/rooting.rs
@@ -370,12 +370,21 @@ struct LifoRoot {
 
 impl RootSet {
     pub(crate) fn trace_roots(&mut self, gc_roots_list: &mut GcRootsList) {
+        log::trace!("Begin trace user LIFO roots");
         for root in &mut self.lifo_roots {
-            gc_roots_list.add_root((&mut root.gc_ref).into());
+            unsafe {
+                gc_roots_list.add_root((&mut root.gc_ref).into());
+            }
         }
+        log::trace!("End trace user LIFO roots");
+
+        log::trace!("Begin trace user manual roots");
         for (_id, root) in self.manually_rooted.iter_mut() {
-            gc_roots_list.add_root(root.into());
+            unsafe {
+                gc_roots_list.add_root(root.into());
+            }
         }
+        log::trace!("End trace user manual roots");
     }
 
     /// Enter a LIFO rooting scope.
diff --git a/crates/wasmtime/src/runtime/store.rs b/crates/wasmtime/src/runtime/store.rs
index f54c5846f..c21273fab 100644
--- a/crates/wasmtime/src/runtime/store.rs
+++ b/crates/wasmtime/src/runtime/store.rs
@@ -1691,8 +1691,11 @@ impl StoreOpaque {
                     .expect("we should never use the high 32 bits of an r64");
 
                 if gc_ref.is_some() {
-                    gc_roots_list
-                        .add_wasm_stack_root(SendSyncPtr::new(NonNull::new(stack_slot).unwrap()));
+                    unsafe {
+                        gc_roots_list.add_wasm_stack_root(SendSyncPtr::new(
+                            NonNull::new(stack_slot).unwrap(),
+                        ));
+                    }
                 }
             }
 


Test patch:
diff --git a/cranelift/filetests/filetests/egraph/bitcast-ref.clif b/cranelift/filetests/filetests/egraph/bitcast-ref.clif
new file mode 100644
index 000000000..e6bb67818
--- /dev/null
+++ b/cranelift/filetests/filetests/egraph/bitcast-ref.clif
@@ -0,0 +1,71 @@
+test optimize precise-output
+set opt_level=speed
+target x86_64
+
+function %bitcast_from_r64(i64 vmctx, r64) -> i32 fast {
+    sig0 = (i32) fast
+    fn0 = colocated %foo sig0
+
+block0(v0: i64, v1: r64):
+    v2 = bitcast.i64 v1
+    v3 = ireduce.i32 v2
+
+    call fn0(v3)
+
+    ;; We MUST NOT dedupe the bitcast *from* the reference type, as that would
+    ;; make `v1` no longer live across the `call`, which would mean that it
+    ;; would not show up in the `call`'s safepoint's stack map, which could
+    ;; result in the collector reclaiming an object too early, which is basically
+    ;; a use-after-free bug.
+    v4 = bitcast.i64 v1
+    v5 = ireduce.i32 v4
+
+    return v5
+}
+
+; function %bitcast_from_r64(i64 vmctx, r64) -> i32 fast {
+;     sig0 = (i32) fast
+;     fn0 = colocated %foo sig0
+;
+; block0(v0: i64, v1: r64):
+;     v2 = bitcast.i64 v1
+;     v3 = ireduce.i32 v2
+;     call fn0(v3)
+;     v4 = bitcast.i64 v1
+;     v5 = ireduce.i32 v4
+;     return v5
+; }
+
+function %bitcast_to_r64(i64 vmctx, i32) -> r64 fast {
+    sig0 = (r64) fast
+    fn0 = colocated %foo sig0
+
+block0(v0: i64, v1: i32):
+    v2 = uextend.i64 v1
+    v3 = bitcast.r64 v2
+
+    call fn0(v3)
+
+    ;; On the other hand, it is technically fine to dedupe bitcasts *to*
+    ;; reference types. Doing so extends the live range of the GC reference --
+    ;; potentially adding it to more stack maps than it otherwise would have
+    ;; been in, which means it might unnecessarily survive a GC it otherwise
+    ;; wouldn't have -- but that is okay. Shrinking live ranges of GC
+    ;; references, and removing them from stack maps they otherwise would have
+    ;; been in, is the problematic transformation.
+    v4 = uextend.i64 v1
+    v5 = bitcast.r64 v4
+
+    return v5
+}
+
+; function %bitcast_to_r64(i64 vmctx, i32) -> r64 fast {
+;     sig0 = (r64) fast
+;     fn0 = colocated %foo sig0
+;
+; block0(v0: i64, v1: i32):
+;     v2 = uextend.i64 v1
+;     v3 = bitcast.r64 v2
+;     call fn0(v3)
+;     return v3
+; }
diff --git a/cranelift/filetests/filetests/isa/aarch64/nan-canonicalization.clif b/cranelift/filetests/filetests/isa/aarch64/nan-canonicalization.clif
new file mode 100644
index 000000000..8ffffcdd2
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/aarch64/nan-canonicalization.clif
@@ -0,0 +1,140 @@
+test compile precise-output
+set enable_nan_canonicalization=true 
+target x86_64 sse41
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addps   %xmm0, %xmm1, %xmm0
+;   movl    $2143289344, %r10d
+;   movd    %r10d, %xmm7
+;   shufps  $0, %xmm7, const(0), %xmm7
+;   movdqa  %xmm0, %xmm1
+;   cmpps   $3, %xmm1, %xmm0, %xmm1
+;   movdqa  %xmm0, %xmm2
+;   movdqa  %xmm1, %xmm0
+;   movdqa  %xmm2, %xmm1
+;   pblendvb %xmm1, %xmm7, %xmm1
+;   movdqa  %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addps %xmm1, %xmm0
+;   movl $0x7fc00000, %r10d
+;   movd %r10d, %xmm7
+;   shufps $0, 0x26(%rip), %xmm7
+;   movdqa %xmm0, %xmm1
+;   cmpunordps %xmm0, %xmm1
+;   movdqa %xmm0, %xmm2
+;   movdqa %xmm1, %xmm0
+;   movdqa %xmm2, %xmm1
+;   pblendvb %xmm0, %xmm7, %xmm1
+;   movdqa %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   sarb $0, (%rdi)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addsd   %xmm0, %xmm1, %xmm0
+;   movabsq $9221120237041090560, %r9
+;   movq    %r9, %xmm1
+;   movdqa  %xmm0, %xmm7
+;   cmppd   $3, %xmm7, %xmm0, %xmm7
+;   movdqa  %xmm0, %xmm5
+;   movdqa  %xmm7, %xmm0
+;   pblendvb %xmm5, %xmm1, %xmm5
+;   movdqa  %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addsd %xmm1, %xmm0
+;   movabsq $0x7ff8000000000000, %r9
+;   movq %r9, %xmm1
+;   movdqa %xmm0, %xmm7
+;   cmpunordpd %xmm0, %xmm7
+;   movdqa %xmm0, %xmm5
+;   movdqa %xmm7, %xmm0
+;   pblendvb %xmm0, %xmm1, %xmm5
+;   movdqa %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addss   %xmm0, %xmm1, %xmm0
+;   movl    $2143289344, %r9d
+;   movd    %r9d, %xmm1
+;   movdqa  %xmm0, %xmm7
+;   cmpps   $3, %xmm7, %xmm0, %xmm7
+;   movdqa  %xmm0, %xmm5
+;   movdqa  %xmm7, %xmm0
+;   pblendvb %xmm5, %xmm1, %xmm5
+;   movdqa  %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addss %xmm1, %xmm0
+;   movl $0x7fc00000, %r9d
+;   movd %r9d, %xmm1
+;   movdqa %xmm0, %xmm7
+;   cmpunordps %xmm0, %xmm7
+;   movdqa %xmm0, %xmm5
+;   movdqa %xmm7, %xmm0
+;   pblendvb %xmm0, %xmm1, %xmm5
+;   movdqa %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization-has_v.clif b/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization-has_v.clif
new file mode 100644
index 000000000..0a9a82457
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization-has_v.clif
@@ -0,0 +1,160 @@
+test compile precise-output
+set enable_nan_canonicalization=true
+target riscv64 has_v
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   addi sp,sp,-16
+;   sd ra,8(sp)
+;   sd fp,0(sp)
+;   mv fp,sp
+; block0:
+;   vle8.v v9,16(fp) #avl=16, #vtype=(e8, m1, ta, ma)
+;   vle8.v v11,32(fp) #avl=16, #vtype=(e8, m1, ta, ma)
+;   vfadd.vv v13,v9,v11 #avl=4, #vtype=(e32, m1, ta, ma)
+;   lui a1,523264
+;   fmv.w.x fa2,a1
+;   vfmv.v.f v14,fa2 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmfne.vv v10,v13,v13 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmfne.vv v12,v13,v13 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmor.mm v0,v10,v12 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmerge.vvm v8,v13,v14,v0.t #avl=4, #vtype=(e32, m1, ta, ma)
+;   vse8.v v8,0(a0) #avl=16, #vtype=(e8, m1, ta, ma)
+;   ld ra,8(sp)
+;   ld fp,0(sp)
+;   addi sp,sp,16
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   addi sp, sp, -0x10
+;   sd ra, 8(sp)
+;   sd s0, 0(sp)
+;   mv s0, sp
+; block1: ; offset 0x10
+;   .byte 0x57, 0x70, 0x08, 0xcc
+;   addi t6, s0, 0x10
+;   .byte 0x87, 0x84, 0x0f, 0x02
+;   addi t6, s0, 0x20
+;   .byte 0x87, 0x85, 0x0f, 0x02
+;   .byte 0x57, 0x70, 0x02, 0xcd
+;   .byte 0xd7, 0x96, 0x95, 0x02
+;   lui a1, 0x7fc00
+;   fmv.w.x fa2, a1
+;   .byte 0x57, 0x57, 0x06, 0x5e
+;   .byte 0x57, 0x95, 0xd6, 0x72
+;   .byte 0x57, 0x96, 0xd6, 0x72
+;   .byte 0x57, 0x20, 0xa6, 0x6a
+;   .byte 0x57, 0x04, 0xd7, 0x5c
+;   .byte 0x57, 0x70, 0x08, 0xcc
+;   .byte 0x27, 0x04, 0x05, 0x02
+;   ld ra, 8(sp)
+;   ld s0, 0(sp)
+;   addi sp, sp, 0x10
+;   ret
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   fadd.d fa1,fa0,fa1,rne
+;   lui a5,4095
+;   slli a1,a5,39
+;   fmv.d.x fa3,a1
+;   vmv.v.x v8,zero #avl=2, #vtype=(e64, m1, ta, ma)
+;   vfmv.s.f v10,fa3 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmv.v.i v0,1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v14,v8,v10,v0.t #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmv.v.x v8,zero #avl=2, #vtype=(e64, m1, ta, ma)
+;   vfmv.s.f v10,fa1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmv.v.i v0,1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v15,v8,v10,v0.t #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmfne.vv v8,v15,v15 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmfne.vv v10,v15,v15 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmor.mm v0,v8,v10 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v8,v15,v14,v0.t #avl=2, #vtype=(e64, m1, ta, ma)
+;   vfmv.f.s fa0,v8 #avl=2, #vtype=(e64, m1, ta, ma)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fadd.d fa1, fa0, fa1, rne
+;   lui a5, 0xfff
+;   slli a1, a5, 0x27
+;   fmv.d.x fa3, a1
+;   .byte 0x57, 0x70, 0x81, 0xcd
+;   .byte 0x57, 0x44, 0x00, 0x5e
+;   .byte 0x57, 0xd5, 0x06, 0x42
+;   .byte 0x57, 0xb0, 0x00, 0x5e
+;   .byte 0x57, 0x07, 0x85, 0x5c
+;   .byte 0x57, 0x44, 0x00, 0x5e
+;   .byte 0x57, 0xd5, 0x05, 0x42
+;   .byte 0x57, 0xb0, 0x00, 0x5e
+;   .byte 0xd7, 0x07, 0x85, 0x5c
+;   .byte 0x57, 0x94, 0xf7, 0x72
+;   .byte 0x57, 0x95, 0xf7, 0x72
+;   .byte 0x57, 0x20, 0x85, 0x6a
+;   .byte 0x57, 0x04, 0xf7, 0x5c
+;   .byte 0x57, 0x15, 0x80, 0x42
+;   ret
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   fadd.s fa0,fa0,fa1,rne
+;   lui a5,523264
+;   fmv.w.x fa1,a5
+;   vmv.v.x v15,zero #avl=4, #vtype=(e32, m1, ta, ma)
+;   vfmv.s.f v9,fa1 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmv.v.i v0,1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v13,v15,v9,v0.t #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmv.v.x v15,zero #avl=4, #vtype=(e32, m1, ta, ma)
+;   vfmv.s.f v9,fa0 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmv.v.i v0,1 #avl=2, #vtype=(e64, m1, ta, ma)
+;   vmerge.vvm v14,v15,v9,v0.t #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmfne.vv v15,v14,v14 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmfne.vv v9,v14,v14 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmor.mm v0,v15,v9 #avl=4, #vtype=(e32, m1, ta, ma)
+;   vmerge.vvm v15,v14,v13,v0.t #avl=4, #vtype=(e32, m1, ta, ma)
+;   vfmv.f.s fa0,v15 #avl=4, #vtype=(e32, m1, ta, ma)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fadd.s fa0, fa0, fa1, rne
+;   lui a5, 0x7fc00
+;   fmv.w.x fa1, a5
+;   .byte 0x57, 0x70, 0x02, 0xcd
+;   .byte 0xd7, 0x47, 0x00, 0x5e
+;   .byte 0xd7, 0xd4, 0x05, 0x42
+;   .byte 0x57, 0x70, 0x81, 0xcd
+;   .byte 0x57, 0xb0, 0x00, 0x5e
+;   .byte 0x57, 0x70, 0x02, 0xcd
+;   .byte 0xd7, 0x86, 0xf4, 0x5c
+;   .byte 0xd7, 0x47, 0x00, 0x5e
+;   .byte 0xd7, 0x54, 0x05, 0x42
+;   .byte 0x57, 0x70, 0x81, 0xcd
+;   .byte 0x57, 0xb0, 0x00, 0x5e
+;   .byte 0x57, 0x70, 0x02, 0xcd
+;   .byte 0x57, 0x87, 0xf4, 0x5c
+;   .byte 0xd7, 0x17, 0xe7, 0x72
+;   .byte 0xd7, 0x14, 0xe7, 0x72
+;   .byte 0x57, 0xa0, 0xf4, 0x6a
+;   .byte 0xd7, 0x87, 0xe6, 0x5c
+;   .byte 0x57, 0x15, 0xf0, 0x42
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization.clif b/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization.clif
new file mode 100644
index 000000000..8f74386af
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/riscv64/nan-canonicalization.clif
@@ -0,0 +1,66 @@
+test compile precise-output
+set enable_nan_canonicalization=true 
+target riscv64
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   fadd.d fa4,fa0,fa1,rne
+;   lui a2,4095
+;   slli a4,a2,39
+;   fmv.d.x fa0,a4
+;   feq.d a1,fa4,fa4
+;   feq.d a3,fa4,fa4
+;   and a5,a1,a3
+;   select fa0,fa0,fa4##condition=(a5 eq zero)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fadd.d fa4, fa0, fa1, rne
+;   lui a2, 0xfff
+;   slli a4, a2, 0x27
+;   fmv.d.x fa0, a4
+;   feq.d a1, fa4, fa4
+;   feq.d a3, fa4, fa4
+;   and a5, a1, a3
+;   beqz a5, 8
+;   fmv.d fa0, fa4
+;   ret
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   fadd.s fa3,fa0,fa1,rne
+;   lui a2,523264
+;   fmv.w.x fa4,a2
+;   feq.s a0,fa3,fa3
+;   feq.s a2,fa3,fa3
+;   and a4,a0,a2
+;   select fa0,fa4,fa3##condition=(a4 eq zero)
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   fadd.s fa3, fa0, fa1, rne
+;   lui a2, 0x7fc00
+;   fmv.w.x fa4, a2
+;   feq.s a0, fa3, fa3
+;   feq.s a2, fa3, fa3
+;   and a4, a0, a2
+;   bnez a4, 0xc
+;   fmv.d fa0, fa4
+;   j 8
+;   fmv.d fa0, fa3
+;   ret
+
diff --git a/cranelift/filetests/filetests/isa/s390x/nan-canonicalization.clif b/cranelift/filetests/filetests/isa/s390x/nan-canonicalization.clif
new file mode 100644
index 000000000..f2619125d
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/s390x/nan-canonicalization.clif
@@ -0,0 +1,112 @@
+test compile precise-output
+set enable_nan_canonicalization=true 
+target s390x
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   vfasb %v17, %v24, %v25
+;   bras %r1, 8 ; data.f32 NaN ; vlef %v18, 0(%r1), 0
+;   vrepf %v18, %v18, 0
+;   vfchesb %v7, %v17, %v17
+;   vfchesb %v19, %v17, %v17
+;   vno %v19, %v7, %v19
+;   vsel %v24, %v18, %v17, %v19
+;   br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   vfasb %v17, %v24, %v25
+;   bras %r1, 0xe
+;   su %f12, 0
+;   vlef %v18, 0(%r1), 0
+;   vrepf %v18, %v18, 0
+;   vfchesb %v7, %v17, %v17
+;   vfchesb %v19, %v17, %v17
+;   vno %v19, %v7, %v19
+;   vsel %v24, %v18, %v17, %v19
+;   br %r14
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   wfadb %v21, %f0, %f2
+;   bras %r1, 12 ; data.f64 NaN ; vleg %v22, 0(%r1), 0
+;   vgbm %v20, 0
+;   vpdi %v22, %v22, %v20, 0
+;   vgbm %v20, 0
+;   vpdi %v23, %v21, %v20, 0
+;   vfchedb %v19, %v23, %v23
+;   vfchedb %v21, %v23, %v23
+;   vno %v24, %v19, %v21
+;   vsel %v21, %v22, %v23, %v24
+;   vrepg %v0, %v21, 0
+;   br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   wfadb %v21, %f0, %f2
+;   bras %r1, 0x12
+;   su %f15, 0(%r8)
+;   .byte 0x00, 0x00
+;   .byte 0x00, 0x00
+;   vleg %v22, 0(%r1), 0
+;   vzero %v20
+;   vpdi %v22, %v22, %v20, 0
+;   vzero %v20
+;   vpdi %v23, %v21, %v20, 0
+;   vfchedb %v19, %v23, %v23
+;   vfchedb %v21, %v23, %v23
+;   vno %v24, %v19, %v21
+;   vsel %v21, %v22, %v23, %v24
+;   vrepg %v0, %v21, 0
+;   br %r14
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+; block0:
+;   wfasb %v21, %f0, %f2
+;   bras %r1, 8 ; data.f32 NaN ; vlef %v22, 0(%r1), 0
+;   vgbm %v20, 61440
+;   vn %v22, %v22, %v20
+;   vgbm %v20, 61440
+;   vn %v23, %v21, %v20
+;   vfchesb %v19, %v23, %v23
+;   vfchesb %v21, %v23, %v23
+;   vno %v24, %v19, %v21
+;   vsel %v21, %v22, %v23, %v24
+;   vrepf %v0, %v21, 0
+;   br %r14
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   wfasb %v21, %f0, %f2
+;   bras %r1, 0xe
+;   su %f12, 0
+;   vlef %v22, 0(%r1), 0
+;   vgbm %v20, 0xf000
+;   vn %v22, %v22, %v20
+;   vgbm %v20, 0xf000
+;   vn %v23, %v21, %v20
+;   vfchesb %v19, %v23, %v23
+;   vfchesb %v21, %v23, %v23
+;   vno %v24, %v19, %v21
+;   vsel %v21, %v22, %v23, %v24
+;   vrepf %v0, %v21, 0
+;   br %r14
+
diff --git a/cranelift/filetests/filetests/isa/x64/nan-canonicalization-sse41.clif b/cranelift/filetests/filetests/isa/x64/nan-canonicalization-sse41.clif
new file mode 100644
index 000000000..b29003307
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/nan-canonicalization-sse41.clif
@@ -0,0 +1,140 @@
+test compile precise-output
+set enable_nan_canonicalization=true
+target x86_64 sse41
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addps   %xmm0, %xmm1, %xmm0
+;   movl    $2143289344, %r10d
+;   movd    %r10d, %xmm7
+;   shufps  $0, %xmm7, const(0), %xmm7
+;   movdqa  %xmm0, %xmm1
+;   cmpps   $3, %xmm1, %xmm0, %xmm1
+;   movdqa  %xmm0, %xmm2
+;   movdqa  %xmm1, %xmm0
+;   movdqa  %xmm2, %xmm1
+;   pblendvb %xmm1, %xmm7, %xmm1
+;   movdqa  %xmm1, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addps %xmm1, %xmm0
+;   movl $0x7fc00000, %r10d
+;   movd %r10d, %xmm7
+;   shufps $0, 0x26(%rip), %xmm7
+;   movdqa %xmm0, %xmm1
+;   cmpunordps %xmm0, %xmm1
+;   movdqa %xmm0, %xmm2
+;   movdqa %xmm1, %xmm0
+;   movdqa %xmm2, %xmm1
+;   pblendvb %xmm0, %xmm7, %xmm1
+;   movdqa %xmm1, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   sarb $0, (%rdi)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addsd   %xmm0, %xmm1, %xmm0
+;   movabsq $9221120237041090560, %r9
+;   movq    %r9, %xmm1
+;   movdqa  %xmm0, %xmm7
+;   cmppd   $3, %xmm7, %xmm0, %xmm7
+;   movdqa  %xmm0, %xmm5
+;   movdqa  %xmm7, %xmm0
+;   pblendvb %xmm5, %xmm1, %xmm5
+;   movdqa  %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addsd %xmm1, %xmm0
+;   movabsq $0x7ff8000000000000, %r9
+;   movq %r9, %xmm1
+;   movdqa %xmm0, %xmm7
+;   cmpunordpd %xmm0, %xmm7
+;   movdqa %xmm0, %xmm5
+;   movdqa %xmm7, %xmm0
+;   pblendvb %xmm0, %xmm1, %xmm5
+;   movdqa %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addss   %xmm0, %xmm1, %xmm0
+;   movl    $2143289344, %r9d
+;   movd    %r9d, %xmm1
+;   movdqa  %xmm0, %xmm7
+;   cmpps   $3, %xmm7, %xmm0, %xmm7
+;   movdqa  %xmm0, %xmm5
+;   movdqa  %xmm7, %xmm0
+;   pblendvb %xmm5, %xmm1, %xmm5
+;   movdqa  %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addss %xmm1, %xmm0
+;   movl $0x7fc00000, %r9d
+;   movd %r9d, %xmm1
+;   movdqa %xmm0, %xmm7
+;   cmpunordps %xmm0, %xmm7
+;   movdqa %xmm0, %xmm5
+;   movdqa %xmm7, %xmm0
+;   pblendvb %xmm0, %xmm1, %xmm5
+;   movdqa %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
diff --git a/cranelift/filetests/filetests/isa/x64/nan-canonicalization.clif b/cranelift/filetests/filetests/isa/x64/nan-canonicalization.clif
new file mode 100644
index 000000000..7454b82fa
--- /dev/null
+++ b/cranelift/filetests/filetests/isa/x64/nan-canonicalization.clif
@@ -0,0 +1,131 @@
+test compile precise-output
+set enable_nan_canonicalization=true 
+target x86_64
+
+function %f0(f32x4, f32x4) -> f32x4 {
+block0(v0: f32x4, v1: f32x4):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addps   %xmm0, %xmm1, %xmm0
+;   movdqa  %xmm0, %xmm1
+;   movl    $2143289344, %esi
+;   movd    %esi, %xmm5
+;   shufps  $0, %xmm5, const(0), %xmm5
+;   cmpps   $3, %xmm0, %xmm1, %xmm0
+;   andps   %xmm5, %xmm0, %xmm5
+;   andnps  %xmm0, %xmm1, %xmm0
+;   orps    %xmm0, %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addps %xmm1, %xmm0
+;   movdqa %xmm0, %xmm1
+;   movl $0x7fc00000, %esi
+;   movd %esi, %xmm5
+;   shufps $0, 0x14(%rip), %xmm5
+;   cmpunordps %xmm1, %xmm0
+;   andps %xmm0, %xmm5
+;   andnps %xmm1, %xmm0
+;   orps %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   sarb $0, (%rdi)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+;   addb %al, (%rax)
+
+function %f1(f64, f64) -> f64 {
+block0(v0: f64, v1: f64):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addsd   %xmm0, %xmm1, %xmm0
+;   movdqa  %xmm0, %xmm7
+;   movabsq $9221120237041090560, %r11
+;   movq    %r11, %xmm5
+;   cmppd   $3, %xmm0, %xmm7, %xmm0
+;   andpd   %xmm5, %xmm0, %xmm5
+;   andnpd  %xmm0, %xmm7, %xmm0
+;   orpd    %xmm0, %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addsd %xmm1, %xmm0
+;   movdqa %xmm0, %xmm7
+;   movabsq $0x7ff8000000000000, %r11
+;   movq %r11, %xmm5
+;   cmpunordpd %xmm7, %xmm0
+;   andpd %xmm0, %xmm5
+;   andnpd %xmm7, %xmm0
+;   orpd %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+
+function %f1(f32, f32) -> f32 {
+block0(v0: f32, v1: f32):
+	v2 = fadd v0, v1
+	return v2
+}
+
+; VCode:
+;   pushq   %rbp
+;   movq    %rsp, %rbp
+; block0:
+;   addss   %xmm0, %xmm1, %xmm0
+;   movdqa  %xmm0, %xmm7
+;   movl    $2143289344, %r11d
+;   movd    %r11d, %xmm5
+;   cmpps   $3, %xmm0, %xmm7, %xmm0
+;   andps   %xmm5, %xmm0, %xmm5
+;   andnps  %xmm0, %xmm7, %xmm0
+;   orps    %xmm0, %xmm5, %xmm0
+;   movq    %rbp, %rsp
+;   popq    %rbp
+;   ret
+;
+; Disassembled:
+; block0: ; offset 0x0
+;   pushq %rbp
+;   movq %rsp, %rbp
+; block1: ; offset 0x4
+;   addss %xmm1, %xmm0
+;   movdqa %xmm0, %xmm7
+;   movl $0x7fc00000, %r11d
+;   movd %r11d, %xmm5
+;   cmpunordps %xmm7, %xmm0
+;   andps %xmm0, %xmm5
+;   andnps %xmm7, %xmm0
+;   orps %xmm5, %xmm0
+;   movq %rbp, %rsp
+;   popq %rbp
+;   retq
+