Repository: bitshifter/glam-rs
Pull request #162 (2021-04-12 10:12:25)

Optimize Affine3D using 4x Vec3A
This makes the SSE vector transforms faster, which argueably is the most important. But a lot of other numbers gets slightly worse (including memory usage).


| op                    | old sse2 | new sse2 |   | old scalar | new scalar |
| --------------------- | -------- | -------- | - | ---------- | ---------- |
| `inverse`             | `9.3 ns` | `8.4 ns` 🥇 |   |  `16 ns`   | `16 ns`   |
| `Affine3D * Affine3D` | `4.2 ns` 🥇 | `5.2 ns` |   |  `13 ns`   | `11 ns` 🥇   |
| `transform point3`    | `3.8 ns` | `2.7 ns` 🥇 |   |  `3.5 ns` 🥇  | `4.0 ns`   |
| `transform vector3`   | `3.6 ns` | `2.5 ns` 🥇 |   |  `2.9 ns` 🥇  | `3.6 ns`   |
| size                  | 48 bytes 🥇 | 64 bytes |   | 48 bytes 🥇 | 64 bytes |


bitshifter: I guess the scalar multiply is losing out due to the alignment/padding :(

Patch:
diff --git a/src/affine3d.rs b/src/affine3d.rs
index c9c5fdc..632a540 100644
--- a/src/affine3d.rs
+++ b/src/affine3d.rs
@@ -1,28 +1,16 @@
-use crate::{Mat3, Mat4, Quat, Vec3, Vec4};
+use crate::{Mat3, Mat4, Quat, Vec3, Vec3A};
 
 #[cfg(not(feature = "std"))]
 use num_traits::Float;
 
-#[cfg(all(
-    target_feature = "sse2",
-    not(feature = "scalar-math"),
-    target_arch = "x86"
-))]
-use core::arch::x86::*;
-#[cfg(all(
-    target_feature = "sse2",
-    not(feature = "scalar-math"),
-    target_arch = "x86_64"
-))]
-use core::arch::x86_64::*;
-
 /// An affine transform, which can represent translation, rotation, scaling and shear.
 #[derive(Copy, Clone, PartialEq)]
 pub struct Affine3D {
-    // TODO: explore different representations, such as 4x Vec3A or Mat3 + Vec3.
-    x_row: Vec4,
-    y_row: Vec4,
-    z_row: Vec4,
+    x_col: Vec3A,
+    y_col: Vec3A,
+    z_col: Vec3A,
+    /// The translation
+    w_col: Vec3A,
 }
 
 impl Default for Affine3D {
@@ -38,50 +26,33 @@ impl Affine3D {
     /// This transforms any finite vector and point to zero.
     /// The zero transform is non-invertible.
     pub const ZERO: Self = Self {
-        x_row: Vec4::ZERO,
-        y_row: Vec4::ZERO,
-        z_row: Vec4::ZERO,
+        x_col: Vec3A::ZERO,
+        y_col: Vec3A::ZERO,
+        z_col: Vec3A::ZERO,
+        w_col: Vec3A::ZERO,
     };
 
     /// The identity transform.
     ///
     /// Multiplying a vector with this returns the same vector.
     pub const IDENTITY: Self = Self {
-        x_row: Vec4::X,
-        y_row: Vec4::Y,
-        z_row: Vec4::Z,
+        x_col: Vec3A::X,
+        y_col: Vec3A::Y,
+        z_col: Vec3A::Z,
+        w_col: Vec3A::ZERO,
     };
 
-    /// Construct a matrix from its three rows.
-    #[inline(always)]
-    fn from_rows(x_row: Vec4, y_row: Vec4, z_row: Vec4) -> Self {
-        Self {
-            x_row,
-            y_row,
-            z_row,
-        }
-    }
-
-    /// Construct a matrix from its four columns.
-    #[inline(always)]
-    fn from_cols(x_col: Vec3, y_col: Vec3, z_col: Vec3, w_col: Vec3) -> Self {
-        Self {
-            x_row: Vec4::new(x_col.x, y_col.x, z_col.x, w_col.x),
-            y_row: Vec4::new(x_col.y, y_col.y, z_col.y, w_col.y),
-            z_row: Vec4::new(x_col.z, y_col.z, z_col.z, w_col.z),
-        }
-    }
-
     /// Creates a transformation matrix that changes scale.
     /// Note that if any scale is zero the transform will be non-invertible.
     #[inline(always)]
     pub fn from_scale(scale: Vec3) -> Self {
         let [x, y, z] = scale.to_array();
-        Self::from_rows(
-            Vec4::new(x, 0.0, 0.0, 0.0),
-            Vec4::new(0.0, y, 0.0, 0.0),
-            Vec4::new(0.0, 0.0, z, 0.0),
-        )
+        Self {
+            x_col: Vec3A::new(x, 0.0, 0.0),
+            y_col: Vec3A::new(0.0, y, 0.0),
+            z_col: Vec3A::new(0.0, 0.0, z),
+            w_col: Vec3A::ZERO,
+        }
     }
 
     /// Creates an affine transformation matrix from the given `rotation` quaternion.
@@ -109,13 +80,11 @@ impl Affine3D {
         let wy = w * y2;
         let wz = w * z2;
 
-        let x_row = Vec4::new(1.0 - (yy + zz), xy - wz, xz + wy, 0.0);
-        let y_row = Vec4::new(xy + wz, 1.0 - (xx + zz), yz - wx, 0.0);
-        let z_row = Vec4::new(xz - wy, yz + wx, 1.0 - (xx + yy), 0.0);
         Self {
-            x_row,
-            y_row,
-            z_row,
+            x_col: Vec3A::new(1.0 - (yy + zz), xy + wz, xz - wy),
+            y_col: Vec3A::new(xy - wz, 1.0 - (xx + zz), yz + wx),
+            z_col: Vec3A::new(xz + wy, yz - wx, 1.0 - (xx + yy)),
+            w_col: Vec3A::ZERO,
         }
     }
 
@@ -134,26 +103,24 @@ impl Affine3D {
         let xyomc = axis.x * axis.y * omc;
         let xzomc = axis.x * axis.z * omc;
         let yzomc = axis.y * axis.z * omc;
-        Self::from_rows(
-            Vec4::new(
+        Self {
+            x_col: Vec3A::new(
                 axis_sq.x * omc + cos,
-                xyomc - axis_sin.z,
+                xyomc + axis_sin.z,
                 xzomc - axis_sin.y,
-                0.0,
             ),
-            Vec4::new(
-                xyomc + axis_sin.z,
+            y_col: Vec3A::new(
+                xyomc - axis_sin.z,
                 axis_sq.y * omc + cos,
                 yzomc + axis_sin.x,
-                0.0,
             ),
-            Vec4::new(
-                xzomc - axis_sin.y,
+            z_col: Vec3A::new(
+                xzomc + axis_sin.y,
                 yzomc - axis_sin.x,
                 axis_sq.z * omc + cos,
-                0.0,
             ),
-        )
+            w_col: Vec3A::ZERO,
+        }
     }
 
     /// Creates an affine transformation matrix containing a 3D rotation around the x axis of
@@ -164,11 +131,12 @@ impl Affine3D {
     #[inline]
     pub fn from_rotation_x(angle: f32) -> Self {
         let (sina, cosa) = angle.sin_cos();
-        Self::from_rows(
-            Vec4::new(1.0, 0.0, 0.0, 0.0),
-            Vec4::new(0.0, cosa, -sina, 0.0),
-            Vec4::new(0.0, sina, cosa, 0.0),
-        )
+        Self {
+            x_col: Vec3A::X,
+            y_col: Vec3A::new(0.0, cosa, sina),
+            z_col: Vec3A::new(0.0, -sina, cosa),
+            w_col: Vec3A::ZERO,
+        }
     }
 
     /// Creates an affine transformation matrix containing a 3D rotation around the y axis of
@@ -179,11 +147,12 @@ impl Affine3D {
     #[inline]
     pub fn from_rotation_y(angle: f32) -> Self {
         let (sina, cosa) = angle.sin_cos();
-        Self::from_rows(
-            Vec4::new(cosa, 0.0, sina, 0.0),
-            Vec4::new(0.0, 1.0, 0.0, 0.0),
-            Vec4::new(-sina, 0.0, cosa, 0.0),
-        )
+        Self {
+            x_col: Vec3A::new(cosa, 0.0, -sina),
+            y_col: Vec3A::Y,
+            z_col: Vec3A::new(sina, 0.0, cosa),
+            w_col: Vec3A::ZERO,
+        }
     }
 
     /// Creates an affine transformation matrix containing a 3D rotation around the z axis of
@@ -194,11 +163,12 @@ impl Affine3D {
     #[inline]
     pub fn from_rotation_z(angle: f32) -> Self {
         let (sina, cosa) = angle.sin_cos();
-        Self::from_rows(
-            Vec4::new(cosa, -sina, 0.0, 0.0),
-            Vec4::new(sina, cosa, 0.0, 0.0),
-            Vec4::new(0.0, 0.0, 1.0, 0.0),
-        )
+        Self {
+            x_col: Vec3A::new(cosa, sina, 0.0),
+            y_col: Vec3A::new(-sina, cosa, 0.0),
+            z_col: Vec3A::Z,
+            w_col: Vec3A::ZERO,
+        }
     }
 
     /// Creates an affine transformation from the given 3D `translation`.
@@ -207,21 +177,22 @@ impl Affine3D {
     /// See [`Self::transform_point3()`] and [`Self::transform_vector3()`].
     #[inline(always)]
     pub fn from_translation(translation: Vec3) -> Self {
-        Self::from_rows(
-            Vec4::new(1.0, 0.0, 0.0, translation.x),
-            Vec4::new(0.0, 1.0, 0.0, translation.y),
-            Vec4::new(0.0, 0.0, 1.0, translation.z),
-        )
+        Self {
+            x_col: Vec3A::X,
+            y_col: Vec3A::Y,
+            z_col: Vec3A::Z,
+            w_col: translation.into(),
+        }
     }
 
     /// Creates an affine transformation from a `Mat3` (expressing scale, shear and rotation)
     #[inline(always)]
     pub fn from_mat3(mat3: Mat3) -> Self {
-        let t = mat3.transpose();
         Self {
-            x_row: t.x_axis.extend(0.0),
-            y_row: t.y_axis.extend(0.0),
-            z_row: t.z_axis.extend(0.0),
+            x_col: mat3.x_axis.into(),
+            y_col: mat3.y_axis.into(),
+            z_col: mat3.z_axis.into(),
+            w_col: Vec3A::ZERO,
         }
     }
 
@@ -234,11 +205,11 @@ impl Affine3D {
     /// See [`Self::transform_point3()`] and [`Self::transform_vector3()`].
     #[inline(always)]
     pub fn from_mat3_translation(mat3: Mat3, translation: Vec3) -> Self {
-        let t = mat3.transpose();
         Self {
-            x_row: t.x_axis.extend(translation.x),
-            y_row: t.y_axis.extend(translation.y),
-            z_row: t.z_axis.extend(translation.z),
+            x_col: mat3.x_axis.into(),
+            y_col: mat3.y_axis.into(),
+            z_col: mat3.z_axis.into(),
+            w_col: translation.into(),
         }
     }
 
@@ -251,14 +222,11 @@ impl Affine3D {
     /// See [`Self::transform_point3()`] and [`Self::transform_vector3()`].
     #[inline(always)]
     pub fn from_scale_rotation_translation(scale: Vec3, rotation: Quat, translation: Vec3) -> Self {
-        let scale4 = scale.extend(0.0);
         let mut m = Self::from_quat(rotation);
-        m.x_row *= scale4;
-        m.y_row *= scale4;
-        m.z_row *= scale4;
-        m.x_row.w = translation.x;
-        m.y_row.w = translation.y;
-        m.z_row.w = translation.z;
+        m.x_col *= scale.x;
+        m.y_col *= scale.y;
+        m.z_col *= scale.z;
+        m.w_col = translation.into();
         m
     }
 
@@ -271,9 +239,7 @@ impl Affine3D {
     #[inline(always)]
     pub fn from_rotation_translation(rotation: Quat, translation: Vec3) -> Self {
         let mut m = Self::from_quat(rotation);
-        m.x_row.w = translation.x;
-        m.y_row.w = translation.y;
-        m.z_row.w = translation.z;
+        m.w_col = translation.into();
         m
     }
 
@@ -281,90 +247,58 @@ impl Affine3D {
     /// i.e. contain no persepctive transform.
     #[inline]
     pub fn from_mat4(m: Mat4) -> Self {
-        Self::from_cols(
-            m.x_axis.truncate(),
-            m.y_axis.truncate(),
-            m.z_axis.truncate(),
-            m.w_axis.truncate(),
-        )
+        Self {
+            x_col: m.x_axis.into(),
+            y_col: m.y_axis.into(),
+            z_col: m.z_axis.into(),
+            w_col: m.w_axis.into(),
+        }
     }
 
     /// The first column.
     #[inline(always)]
     fn x_col(&self) -> Vec3 {
-        Vec3::new(self.x_row.x, self.y_row.x, self.z_row.x)
+        self.x_col.into()
     }
 
     /// The second column.
     #[inline(always)]
     fn y_col(&self) -> Vec3 {
-        Vec3::new(self.x_row.y, self.y_row.y, self.z_row.y)
+        self.y_col.into()
     }
 
     /// The thirs column.
     #[inline(always)]
     fn z_col(&self) -> Vec3 {
-        Vec3::new(self.x_row.z, self.y_row.z, self.z_row.z)
+        self.z_col.into()
     }
 
     /// The translation expressed by this transform.
     /// The translation is applied last, so is separatable from scale, shear and rotation.
     #[inline(always)]
     pub fn translation(&self) -> Vec3 {
-        Vec3::new(self.x_row.w, self.y_row.w, self.z_row.w)
+        self.w_col.into()
     }
 
     /// Set the translation part of this transform.
     /// The translation is applied last, so is separatable from scale, shear and rotation.
     #[inline(always)]
     pub fn set_translation(&mut self, translation: Vec3) {
-        self.x_row.w = translation.x;
-        self.y_row.w = translation.y;
-        self.z_row.w = translation.z;
+        self.w_col = translation.into();
     }
 
     /// The scale, shear and rotation expressed by this transform.
     #[inline(always)]
     pub fn mat3(&self) -> Mat3 {
-        Mat3::from_cols(
-            self.x_row.truncate(),
-            self.y_row.truncate(),
-            self.z_row.truncate(),
-        )
-        .transpose()
+        Mat3::from_cols(self.x_col.into(), self.y_col.into(), self.z_col.into())
     }
 
     /// Set the scale, shear and rotation expressed by this transform.
     #[inline(always)]
-    pub fn set_mat3(&mut self, mat3: Mat3) {
-        let t = mat3.transpose();
-        self.x_row = t.x_axis.extend(self.x_row.w);
-        self.y_row = t.y_axis.extend(self.y_row.w);
-        self.z_row = t.z_axis.extend(self.z_row.w);
-    }
-
-    /// The first column extended with `0`.
-    #[inline(always)]
-    fn x_col_extended_0(&self) -> Vec4 {
-        Vec4::new(self.x_row.x, self.y_row.x, self.z_row.x, 0.0)
-    }
-
-    /// The second column extended with `0`.
-    #[inline(always)]
-    fn y_col_extended_0(&self) -> Vec4 {
-        Vec4::new(self.x_row.y, self.y_row.y, self.z_row.y, 0.0)
-    }
-
-    /// The thirs column extended with `0`.
-    #[inline(always)]
-    fn z_col_extended_0(&self) -> Vec4 {
-        Vec4::new(self.x_row.z, self.y_row.z, self.z_row.z, 0.0)
-    }
-
-    /// The fourth column extended with `1`.
-    #[inline(always)]
-    fn w_col_extended_1(&self) -> Vec4 {
-        Vec4::new(self.x_row.w, self.y_row.w, self.z_row.w, 1.0)
+    pub fn set_mat3(&mut self, m: Mat3) {
+        self.x_col = m.x_axis.into();
+        self.y_col = m.y_axis.into();
+        self.z_col = m.z_axis.into();
     }
 
     /// Extracts `scale`, `rotation` and `translation` from `self`.
@@ -402,46 +336,25 @@ impl Affine3D {
     #[inline]
     fn determinant(&self) -> f32 {
         // As if the last row was `[0, 0, 0, 1]`.
-        let [m00, m01, m02, _] = self.x_row.to_array();
-        let [m10, m11, m12, _] = self.y_row.to_array();
-        let [m20, m21, m22, _] = self.z_row.to_array();
+        let [m00, m10, m20] = self.x_col.to_array();
+        let [m01, m11, m21] = self.y_col.to_array();
+        let [m02, m12, m22] = self.z_col.to_array();
 
         m00 * m11 * m22 - m00 * m12 * m21 - m01 * m10 * m22 + m01 * m12 * m20 + m02 * m10 * m21
             - m02 * m11 * m20
     }
 
-    /// Multiply with a column vector (`m * v`)
-    #[inline(always)]
-    fn mul_vec4(&self, rhs: Vec4) -> Vec3 {
-        Vec3::new(
-            self.x_row.dot(rhs),
-            self.y_row.dot(rhs),
-            self.z_row.dot(rhs),
-        )
-    }
-
-    /// Multiply with a row vector (`x * m`)
-    /// as if we were a 4x4 matrix (implict `[0, 0, 0, 1]` last row)
-    #[inline(always)]
-    fn mul_row_vec4_from_left(&self, lhs: Vec4) -> Vec4 {
-        Vec4::new(
-            self.x_col_extended_0().dot(lhs),
-            self.y_col_extended_0().dot(lhs),
-            self.z_col_extended_0().dot(lhs),
-            self.w_col_extended_1().dot(lhs),
-        )
-    }
-
     #[inline]
     fn look_to_lh(eye: Vec3, dir: Vec3, up: Vec3) -> Self {
         let f = dir.normalize();
         let s = up.cross(f).normalize();
         let u = f.cross(s);
-        Self::from_rows(
-            s.extend(-s.dot(eye)),
-            u.extend(-u.dot(eye)),
-            f.extend(-f.dot(eye)),
-        )
+        Self {
+            x_col: Vec3A::new(s.x, u.x, f.x),
+            y_col: Vec3A::new(s.y, u.y, f.y),
+            z_col: Vec3A::new(s.z, u.z, f.z),
+            w_col: Vec3A::new(-s.dot(eye), -u.dot(eye), -f.dot(eye)),
+        }
     }
 
     /// Creates a left-handed view transform using a camera position, an up direction, and a focal
@@ -463,50 +376,33 @@ impl Affine3D {
     }
 
     /// Transforms the given 3D points, applying shear, scale, rotation and translatio.
-    #[inline]
+    #[inline(always)]
     pub fn transform_point3(&self, point: Vec3) -> Vec3 {
-        self.mul_vec4(point.extend(1.0))
-    }
-
-    /// Transforms the give 3D vector, applying shear, scale and rotation (but NOT translation).
-    ///
-    /// To also apply translation, use [`Self::transform_point3`] instead.
-    #[inline]
-    #[cfg(any(
-        target_arch = "spirv",
-        all(target_feature = "sse2", not(feature = "scalar-math"))
-    ))]
-    pub fn transform_vector3(&self, vec: Vec3) -> Vec3 {
-        self.mul_vec4(vec.extend(0.0))
+        (self.x_col * point.x + self.y_col * point.y + self.z_col * point.z + self.w_col).into()
     }
 
     /// Transforms the give 3D vector, applying shear, scale and rotation (but NOT translation).
     ///
     /// To also apply translation, use [`Self::transform_point3`] instead.
-    #[inline]
-    #[cfg(not(any(
-        target_arch = "spirv",
-        all(target_feature = "sse2", not(feature = "scalar-math"))
-    )))]
+    #[inline(always)]
     pub fn transform_vector3(&self, vec: Vec3) -> Vec3 {
-        Vec3::new(
-            Vec3::from(self.x_row).dot(vec),
-            Vec3::from(self.y_row).dot(vec),
-            Vec3::from(self.z_row).dot(vec),
-        )
+        (self.x_col * vec.x + self.y_col * vec.y + self.z_col * vec.z).into()
     }
 
     /// Returns `true` if, and only if, all elements are finite.
     /// If any element is either `NaN`, positive or negative infinity, this will return `false`.
     #[inline]
     pub fn is_finite(&self) -> bool {
-        self.x_row.is_finite() && self.y_row.is_finite() && self.z_row.is_finite()
+        self.x_col.is_finite()
+            && self.y_col.is_finite()
+            && self.z_col.is_finite()
+            && self.w_col.is_finite()
     }
 
     /// Returns `true` if any elements are `NaN`.
     #[inline]
     pub fn is_nan(&self) -> bool {
-        self.x_row.is_nan() || self.y_row.is_nan() || self.z_row.is_nan()
+        self.x_col.is_nan() || self.y_col.is_nan() || self.z_col.is_nan() || self.w_col.is_nan()
     }
 
     /// Returns true if the absolute difference of all elements between `self` and `other`
@@ -520,9 +416,10 @@ impl Affine3D {
     /// [comparing floating point numbers](https://randomascii.wordpress.com/2012/02/25/comparing-floating-point-numbers-2012-edition/).
     #[inline]
     pub fn abs_diff_eq(&self, other: Self, max_abs_diff: f32) -> bool {
-        self.x_row.abs_diff_eq(other.x_row, max_abs_diff)
-            && self.y_row.abs_diff_eq(other.y_row, max_abs_diff)
-            && self.z_row.abs_diff_eq(other.z_row, max_abs_diff)
+        self.x_col.abs_diff_eq(other.x_col, max_abs_diff)
+            && self.y_col.abs_diff_eq(other.y_col, max_abs_diff)
+            && self.z_col.abs_diff_eq(other.z_col, max_abs_diff)
+            && self.w_col.abs_diff_eq(other.w_col, max_abs_diff)
     }
 
     /// Return the inverse of this transform.
@@ -530,27 +427,27 @@ impl Affine3D {
     /// The result of this is only valid if [`Self::is_invertible`] is true.
     pub fn inverse(&self) -> Self {
         // invert 3x3 matrix:
-        use crate::Vec3A;
-        let x_row3: Vec3A = self.x_row.into();
-        let y_row3: Vec3A = self.y_row.into();
-        let z_row3: Vec3A = self.z_row.into();
-
-        let x_col = y_row3.cross(z_row3);
-        let y_col = z_row3.cross(x_row3);
-        let z_col = x_row3.cross(y_row3);
-        let det = z_row3.dot(z_col);
+        let x_row = self.y_col.cross(self.z_col);
+        let y_row = self.z_col.cross(self.x_col);
+        let z_row = self.x_col.cross(self.y_col);
+        let det = self.z_col.dot(z_row);
         let inv_det = det.recip();
-        let x_col = x_col * inv_det;
-        let y_col = y_col * inv_det;
-        let z_col = z_col * inv_det;
+        let x_row = x_row * inv_det;
+        let y_row = y_row * inv_det;
+        let z_row = z_row * inv_det;
+
+        let x_col = Vec3A::new(x_row.x, y_row.x, z_row.x);
+        let y_col = Vec3A::new(x_row.y, y_row.y, z_row.y);
+        let z_col = Vec3A::new(x_row.z, y_row.z, z_row.z);
 
         // transform negative translation by the 3x3 inverse:
-        let w_col = -(x_col * self.x_row.w + y_col * self.y_row.w + z_col * self.z_row.w);
+        let w_col = -(x_col * self.w_col.x + y_col * self.w_col.y + z_col * self.w_col.z);
 
         Self {
-            x_row: Vec4::new(x_col.x, y_col.x, z_col.x, w_col.x),
-            y_row: Vec4::new(x_col.y, y_col.y, z_col.y, w_col.y),
-            z_row: Vec4::new(x_col.z, y_col.z, z_col.z, w_col.z),
+            x_col,
+            y_col,
+            z_col,
+            w_col,
         }
     }
 }
@@ -559,9 +456,10 @@ impl Affine3D {
 impl core::fmt::Debug for Affine3D {
     fn fmt(&self, fmt: &mut core::fmt::Formatter) -> core::fmt::Result {
         fmt.debug_struct(stringify!($affine3d))
-            .field("x_row", &self.x_row)
-            .field("y_row", &self.y_row)
-            .field("z_row", &self.z_row)
+            .field("x_col", &self.x_col)
+            .field("y_col", &self.y_col)
+            .field("z_col", &self.z_col)
+            .field("w_col", &self.w_col)
             .finish()
     }
 }
@@ -570,10 +468,10 @@ impl From<Affine3D> for Mat4 {
     #[inline]
     fn from(m: Affine3D) -> Mat4 {
         Mat4::from_cols(
-            m.x_col_extended_0(),
-            m.y_col_extended_0(),
-            m.z_col_extended_0(),
-            m.w_col_extended_1(),
+            m.x_col.extend(0.0),
+            m.y_col.extend(0.0),
+            m.z_col.extend(0.0),
+            m.w_col.extend(1.0),
         )
     }
 }
@@ -591,59 +489,56 @@ impl core::ops::Mul for Affine3D {
         use core::arch::x86_64::*;
 
         unsafe {
-            let lhs_x_row: __m128 = self.x_row.into();
-            let lhs_y_row: __m128 = self.y_row.into();
-            let lhs_z_row: __m128 = self.z_row.into();
+            // TODO: optimize
+            let lhs_x_col: __m128 = self.x_col.into();
+            let lhs_y_col: __m128 = self.y_col.into();
+            let lhs_z_col: __m128 = self.z_col.into();
+            let lhs_w_col: __m128 = self.w_col.into();
 
-            let rhs_x_row: __m128 = rhs.x_row.into();
-            let rhs_y_row: __m128 = rhs.y_row.into();
-            let rhs_z_row: __m128 = rhs.z_row.into();
-            let rhs_w_row = _mm_set_ps(1.0, 0.0, 0.0, 0.0); // TODO: optimize based on this being `[0, 0, 0, 1]`.
+            let rhs_x_col: __m128 = rhs.x_col.into();
+            let rhs_y_col: __m128 = rhs.y_col.into();
+            let rhs_z_col: __m128 = rhs.z_col.into();
+            let rhs_w_col: __m128 = rhs.w_col.into();
 
             // Based on https://github.com/microsoft/DirectXMath XMMatrixMultiply
 
-            let v_x = _mm_mul_ps(lhs_x_row.splat_x(), rhs_x_row);
-            let v_y = _mm_mul_ps(lhs_x_row.splat_y(), rhs_y_row);
-            let v_z = _mm_mul_ps(lhs_x_row.splat_z(), rhs_z_row);
-            let v_w = _mm_mul_ps(lhs_x_row.splat_w(), rhs_w_row);
-            let out_x_row = _mm_add_ps(_mm_add_ps(v_x, v_z), _mm_add_ps(v_y, v_w));
+            let v_x = _mm_mul_ps(rhs_x_col.splat_x(), lhs_x_col);
+            let v_y = _mm_mul_ps(rhs_x_col.splat_y(), lhs_y_col);
+            let v_z = _mm_mul_ps(rhs_x_col.splat_z(), lhs_z_col);
+            let out_x_col = _mm_add_ps(_mm_add_ps(v_x, v_z), v_y);
+
+            let v_x = _mm_mul_ps(rhs_y_col.splat_x(), lhs_x_col);
+            let v_y = _mm_mul_ps(rhs_y_col.splat_y(), lhs_y_col);
+            let v_z = _mm_mul_ps(rhs_y_col.splat_z(), lhs_z_col);
+            let out_y_col = _mm_add_ps(_mm_add_ps(v_x, v_z), v_y);
 
-            let v_x = _mm_mul_ps(lhs_y_row.splat_x(), rhs_x_row);
-            let v_y = _mm_mul_ps(lhs_y_row.splat_y(), rhs_y_row);
-            let v_z = _mm_mul_ps(lhs_y_row.splat_z(), rhs_z_row);
-            let v_w = _mm_mul_ps(lhs_y_row.splat_w(), rhs_w_row);
-            let out_y_row = _mm_add_ps(_mm_add_ps(v_x, v_z), _mm_add_ps(v_y, v_w));
+            let v_x = _mm_mul_ps(rhs_z_col.splat_x(), lhs_x_col);
+            let v_y = _mm_mul_ps(rhs_z_col.splat_y(), lhs_y_col);
+            let v_z = _mm_mul_ps(rhs_z_col.splat_z(), lhs_z_col);
+            let out_z_col = _mm_add_ps(_mm_add_ps(v_x, v_z), v_y);
 
-            let v_x = _mm_mul_ps(lhs_z_row.splat_x(), rhs_x_row);
-            let v_y = _mm_mul_ps(lhs_z_row.splat_y(), rhs_y_row);
-            let v_z = _mm_mul_ps(lhs_z_row.splat_z(), rhs_z_row);
-            let v_w = _mm_mul_ps(lhs_z_row.splat_w(), rhs_w_row);
-            let out_z_row = _mm_add_ps(_mm_add_ps(v_x, v_z), _mm_add_ps(v_y, v_w));
+            let v_x = _mm_mul_ps(rhs_w_col.splat_x(), lhs_x_col);
+            let v_y = _mm_mul_ps(rhs_w_col.splat_y(), lhs_y_col);
+            let v_z = _mm_mul_ps(rhs_w_col.splat_z(), lhs_z_col);
+            let out_w_col = _mm_add_ps(_mm_add_ps(v_x, v_z), _mm_add_ps(v_y, lhs_w_col));
 
             Self {
-                x_row: out_x_row.into(),
-                y_row: out_y_row.into(),
-                z_row: out_z_row.into(),
+                x_col: out_x_col.into(),
+                y_col: out_y_col.into(),
+                z_col: out_z_col.into(),
+                w_col: out_w_col.into(),
             }
         }
     }
 
-    #[cfg(target_arch = "spirv")]
-    #[inline(always)]
-    fn mul(self, rhs: Affine3D) -> Self::Output {
-        Affine3D::from_mat4(Mat4::from(self) * Mat4::from(rhs))
-    }
-
-    #[cfg(not(any(
-        target_arch = "spirv",
-        all(target_feature = "sse2", not(feature = "scalar-math"))
-    )))]
+    #[cfg(not(all(target_feature = "sse2", not(feature = "scalar-math"))))]
     #[inline(always)]
     fn mul(self, rhs: Affine3D) -> Self::Output {
         Self {
-            x_row: rhs.mul_row_vec4_from_left(self.x_row),
-            y_row: rhs.mul_row_vec4_from_left(self.y_row),
-            z_row: rhs.mul_row_vec4_from_left(self.z_row),
+            x_col: self.transform_vector3(rhs.x_col.into()).into(),
+            y_col: self.transform_vector3(rhs.y_col.into()).into(),
+            z_col: self.transform_vector3(rhs.z_col.into()).into(),
+            w_col: self.transform_point3(rhs.w_col.into()).into(),
         }
     }
 }


Test patch:
diff --git a/tests/affine3d.rs b/tests/affine3d.rs
index 3f64ba7..c24406f 100644
--- a/tests/affine3d.rs
+++ b/tests/affine3d.rs
@@ -240,12 +240,8 @@ mod affine3d {
     #[test]
     fn test_align() {
         use std::mem;
-        assert_eq!(48, mem::size_of::<Affine3D>());
-        if cfg!(feature = "scalar-math") {
-            assert_eq!(4, mem::align_of::<Affine3D>());
-        } else {
-            assert_eq!(16, mem::align_of::<Affine3D>());
-        }
+        assert_eq!(64, mem::size_of::<Affine3D>());
+        assert_eq!(16, mem::align_of::<Affine3D>());
     }
 
     impl_affine3d_tests!(f32, Affine3D, Quat, Vec4, Vec3);