Repository: uxlfoundation/oneDNN
Pull request #2765: [GPU] Enable f4_e3m0 GEMM dst
Author: login: kealan-barbieri
Labels: name: platform:gpu-intel
description: Codeowner: @oneapi-src/onednn-gpu-intel
name: component:tests
description: Codeowner: @oneapi-src/onednn-arch
Created: 2025-02-27T00:59:37Z | Merged: 2025-03-19T16:10:57Z

# Description

Add emulated conversion routine for hf->e3m0, enabling pure jit:gemm with f4_e3m0 dst. 

Partially implements MFDNN-12471

# Checklist

## General

- [x] Do all unit and benchdnn tests (`make test` and `make test_benchdnn_*`) pass locally for each commit?
- [x] Have you formatted the code using clang-format?



Diff:
diff --git a/src/gpu/intel/jit/gemm/gen_gemm.hpp b/src/gpu/intel/jit/gemm/gen_gemm.hpp
index f6d2d3a5b9d..ec358fd6375 100644
--- a/src/gpu/intel/jit/gemm/gen_gemm.hpp
+++ b/src/gpu/intel/jit/gemm/gen_gemm.hpp
@@ -172,9 +172,8 @@ struct gen_gemm_t : public gpu_gemm_t {
                                         && utils::one_of(
                                                 d->b_type(), f8_e5m2, f8_e4m3))
                                 || (utils::one_of(d->a_type(), f4_e2m1, f4_e3m0)
-                                        && utils::one_of(
-                                                d->b_type(), f4_e2m1, f4_e3m0)))
-                                && d->c_type() != f4_e3m0,
+                                        && utils::one_of(d->b_type(), f4_e2m1,
+                                                f4_e3m0))),
                         VERBOSE_INCONSISTENT_DT, "a", "b");
                 VDISPATCH_GEMM(utils::one_of(d->acc_type, d->a_type(), f32),
                         VERBOSE_UNSUPPORTED_DT);
diff --git a/src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp b/src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
index cb10c12e327..9f91765b223 100644
--- a/src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
+++ b/src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
@@ -671,12 +671,18 @@ void CopyPlan::planTypeConversions()
         } else if (st == Type::ngen_f4_e3m0() && dt == DataType::hf) {
             planEmulatedE3M0ToHF(i);
             rerun = true;
-        } else if (dt == Type::ngen_f4_e2m1()) {
+        } else if (dt == Type::ngen_f4_e3m0() && st == DataType::hf) {
+            planEmulatedHFToE3M0(i);
+            rerun = true;
+        } else if (isFP4(dt)) {
             copyThrough(i, DataType::hf);
             rerun = true;
-        } else if (st == Type::ngen_f4_e2m1()) {
+        } else if (isFP4(st)) {
             copyThrough(i, DataType::hf);
             rerun = true;
+        } else if ((isB(st) || isW(st)) && isInt4(dt)) {
+            planBToI4(i);
+            rerun = true;
         } else if (st == DataType::u4 && dt == DataType::hf) {
             copyThrough(i, DataType::uw);
             rerun = true;
@@ -901,7 +907,7 @@ void CopyPlan::planEmulatedHFToF4E2M1(CopyInstruction &i)
     if (i.src0.neg || i.sat || i.hasCMod()) stub("Unsupported modifier");
     int simd = i.simd;
 
-    auto ie = splitMultiple<14>(i);
+    auto ie = splitMultiple<10>(i);
     auto tmp = newTemp(DataType::ud, simd, 1);
 
     auto convFlag = newFlag(simd);
@@ -991,58 +997,108 @@ void CopyPlan::planEmulatedHFToF4E2M1(CopyInstruction &i)
     ie[8]->dst.type = DataType::uw;
     ie[8]->src0 = tmp;
     ie[8]->src0.type = DataType::uw;
-    ie[8]->src1 = Immediate::uw(8);
+    ie[8]->src1 = Immediate::uw(12);
 
     // Pack into byte.
-    ie[9]->op = Opcode::and_;
+
+    ie[9]->op = Opcode::mov;
     ie[9]->simd = simd;
-    ie[9]->dst = tmp;
-    ie[9]->dst.type = DataType::uw;
+    ie[9]->dst = ddst;
+    ie[9]->dst.type = DataType::u4;
     ie[9]->src0 = tmp;
     ie[9]->src0.type = DataType::uw;
-    ie[9]->src1 = Immediate(0x00f0);
-
-    ie[10]->op = Opcode::shr;
-    ie[10]->simd = simd/2;
-    ie[10]->dst = tmp;
-    ie[10]->dst.type = DataType::uw;
-    ie[10]->dst.stride = 2;
-    ie[10]->src0 = tmp;
-    ie[10]->src0.type = DataType::uw;
-    ie[10]->src0.stride = 2;
-    ie[10]->src1 = Immediate::uw(4);
-
-    ie[11]->op = Opcode::or_;
-    ie[11]->simd = simd/2;
-    ie[11]->dst = tmp;
-    ie[11]->dst.type = DataType::uw;
-    ie[11]->src0 = tmp;
-    ie[11]->src0.offset = 1;
-    ie[11]->src0.type = DataType::uw;
-    ie[11]->src0.stride = 2;
-    ie[11]->src1 = tmp;
-    ie[11]->src1.type = DataType::uw;
-    ie[11]->src1.stride = 2;
-
-    ie[12]->op = Opcode::mov;
-    ie[12]->simd = simd/2;
-    ie[12]->dst = tmp;
-    ie[12]->dst.stride = 1;
-    ie[12]->dst.type = DataType::ub;
-    ie[12]->src0 = tmp;
-    ie[12]->src0.stride = 2;
-    ie[12]->src0.type = DataType::ub;
-
-    ie[13]->op = Opcode::mov;
-    ie[13]->simd = simd/2;
-    ie[13]->dst = ddst;
+
+}
+
+void CopyPlan::planBToI4(CopyInstruction &i)
+{
+    if (i.src0.neg || i.sat || i.hasCMod()) stub("Unsupported modifier");
+    int simd = i.simd;
+
+    auto ie = splitMultiple<6>(i);
+    auto tmp = newTemp(DataType::uw, simd, 1);
+
+    auto ddst = CopyOperand(i.dst);
+    auto ssrc = CopyOperand(i.src0);
+    bool sUw = ssrc.type == DataType::uw;
+    bool sUb = ssrc.type == DataType::ub;
+    bool tempSrc = ((ssrc.stride > 1 && ssrc.type == DataType::uw) || (ssrc.stride == 1 && ssrc.type == DataType::ub));
+    if ((!sUw && !sUb)) stub();
+    int idx = 0;
+
+    if(tempSrc){
+        ie[idx]->op = Opcode::mov;
+        ie[idx]->dst = ssrc;
+        ie[idx]->dst.type = DataType::ub;
+        ie[idx]->dst.stride = 2;
+        ie[idx]->src0 = ssrc;
+        if(sUw){
+           ie[idx]->src0.type = DataType::ub;
+           ie[idx]->src0.stride = ie[idx]->src0.stride * 2;
+        }
+    }else{
+        ie[idx]->invalidate();
+    }
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = simd/2;
+    ie[idx]->dst = tmp;
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->src0 = ssrc;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src0.stride = 2;
+    ++idx;
+
+    ie[idx]->op = Opcode::shl;
+    ie[idx]->simd = simd/2;
+    ie[idx]->dst = ssrc;
+    ie[idx]->dst.offset = 0;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = ssrc;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src0.offset = 1;
+    ie[idx]->src0.stride = 2;
+    ie[idx]->src1 = Immediate::uw(0x4);
+    ++idx;
+
+    ie[idx]->op = Opcode::or_;
+    ie[idx]->simd = simd/2;
+    ie[idx]->dst = tmp;
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->src0 = tmp;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src1 = ssrc;
+    ie[idx]->src1.type = DataType::uw;
+    ie[idx]->src1.stride = 1;
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = simd/2;
+    ie[idx]->dst = tmp;
+    ie[idx]->dst.type = DataType::ub;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->src0 = tmp;
+    ie[idx]->src0.stride = 2;
+    ie[idx]->src0.type = DataType::ub;
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = simd/2;
+    ie[idx]->dst = ddst;
+    ie[idx]->dst.type = DataType::ub;
     if (ddst.inVS != 0)
-    ie[13]->dst.stride = ddst.inVS / ddst.inW;
-    ie[13]->dst.type = DataType::ub;
-    if (ie[13]->dst.offset != 0)
-            ie[13]->dst.offset /= 2;
-    ie[13]->src0 = tmp;
-    ie[13]->src0.type = DataType::ub;
+        ie[idx]->dst.stride = ddst.inVS / ddst.inW;
+    if (ie[idx]->dst.offset != 0)
+            ie[idx]->dst.offset /= 2;
+    ie[idx]->src0 = tmp;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::ub;
+    ++idx;
 }
 
 // Emulated f->bf or hf->bf8 sequence.
@@ -1472,6 +1528,153 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // // Emulation sequence for  mov y:e3m0 x:hf
+    // sel (lt)     t0:hf   x:hf    0x4c00:hf       /* clamp max val */
+    // mul          t0:hf   t0:hf   0x0c00:hf       /* round up */
+    // add          t0:uw   t0:uw   -0x200          /* subtract 1/2 ULP */
+    // and (nz)f0   null:uw t0:uw   0x7FF
+    // (f0) add     t0:uw   t0:uw   0x400           /* round up */
+    // shl          t0:uw   t0:uw   2
+    // bfn.0xCA     t0:uw   x:uw    t0:uw  0x8000   /* copy sign */
+    // shr          t0:uw   t0:uw   12
+    // mov /* pack halfs into bytes */
+
+    if (i.src0.neg || i.sat || i.hasCMod()) stub("Unsupported modifier");
+
+    auto ie = splitMultiple<10>(i);
+
+    auto x = i.src0;
+    auto y = i.dst;
+
+    bool tempX = (x.stride > 1);
+
+    auto bits = newTemp(DataType::uw, i.simd, 1, 0, 0);
+
+    auto selFlag = newFlag(i.simd);
+    int simd = i.simd;
+    int idx = 0;
+
+    if(tempX){
+        ie[idx]->op = Opcode::mov;
+        ie[idx]->dst = x;
+        ie[idx]->dst.stride = 1;
+        ie[idx]->src0 = x;
+    }else{
+        ie[idx]->invalidate();
+    }
+    ++idx;
+
+    ie[idx]->op = Opcode::sel;
+    ie[idx]->cmod = ConditionModifier::lt;
+    ie[idx]->dst = bits;
+    ie[idx]->dst.type = DataType::hf;
+    ie[idx]->src0 = x;
+    ie[idx]->src0.abs = true;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::hf;
+    ie[idx]->src1 = 0x4c00;
+    ie[idx]->src1.type = DataType::hf;
+    ++idx;
+
+    ie[idx]->op = Opcode::mul;
+    ie[idx]->dst = bits;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->dst.type = DataType::hf;
+    ie[idx]->src0 = bits;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::hf;
+    ie[idx]->src1 = 0x0c00;
+    ie[idx]->src1.type = DataType::hf;
+    ++idx;
+
+    ie[idx]->op = Opcode::add;
+    ie[idx]->dst = bits;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = bits;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src1 = -0x0200;
+    ie[idx]->src1.type = DataType::w;
+    ++idx;
+
+    ie[idx]->op = Opcode::and_;
+    ie[idx]->flag = selFlag;
+    ie[idx]->cmod = ConditionModifier::nz;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.stride = 1;
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = bits;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src1 = 0x07ff;
+    ie[idx]->src1.type = DataType::uw;
+    ++idx;
+
+    ie[idx]->op = Opcode::add;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = bits;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = bits;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src1 = 0x400;
+    ie[idx]->src1.type = DataType::uw;
+    ++idx;
+
+    ie[idx]->op = Opcode::shl;
+    ie[idx]->dst = bits;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = bits;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src1 = 0x2;
+    ie[idx]->src1.type = DataType::uw;
+    ++idx;
+
+    ie[idx]->op = Opcode::bfn;
+    ie[idx]->dst = bits;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = bits;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src1 = x;
+    ie[idx]->src1.stride = 1;
+    ie[idx]->src1.type = DataType::uw;
+    ie[idx]->src2 = 0x8000;
+    ie[idx]->ctrl = 0xCA;
+    ++idx;
+
+    ie[idx]->op = Opcode::shr;
+    ie[idx]->dst = bits;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = bits;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src1 = 0xC;
+    ie[idx]->src1.type = DataType::uw;
+    ++idx;
+
+    // Pack.
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = simd;
+    ie[idx]->dst = y;
+    ie[idx]->dst.type = DataType::u4;
+    ie[idx]->dst.stride = 1;
+    ie[idx]->src0 = bits;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src0.stride = 1;
+    ++idx;
+
+}
+
 // Emulation sequence for e3m0->hf conversion.
 void CopyPlan::planEmulatedE3M0ToHF(CopyInstruction &i)
 {
diff --git a/src/gpu/intel/jit/gemm/generator/pieces/copy_plan.hpp b/src/gpu/intel/jit/gemm/generator/pieces/copy_plan.hpp
index 99132d334fe..c3bb7ebc677 100644
--- a/src/gpu/intel/jit/gemm/generator/pieces/copy_plan.hpp
+++ b/src/gpu/intel/jit/gemm/generator/pieces/copy_plan.hpp
@@ -134,7 +134,6 @@ class CopyPlan
     using FlagAllocator = std::function<void(int bytes, ngen::FlagRegister &flag)>;
 
     CopyPlan(ngen::HW hw_, bool systolicAvailable_) : hw(hw_), systolicAvailable(systolicAvailable_) {}
-
     CopyInstruction &append(CopyInstruction &&i);
     CopyInstruction &append(ngen::Opcode op, int simd, const CopyOperand &dst, const CopyOperand &src0, const CopyOperand &src1 = CopyOperand(), const CopyOperand &src2 = CopyOperand());
     CopyInstruction &append(ngen::Opcode op, int simd, ngen::InstructionModifier mod, const CopyOperand &dst, const CopyOperand &src0, const CopyOperand &src1 = CopyOperand(), const CopyOperand &src2 = CopyOperand());
@@ -189,9 +188,11 @@ class CopyPlan
     void planEarlyInt4Upconversions();
     void planEmulatedHalveFloat(CopyInstruction &i);
     void planSmallUWToHF(CopyInstruction &i);
+    void planBToI4(CopyInstruction &i);
     void planBToHF(CopyInstruction &i);
     void planS4ToHF(CopyInstruction &i);
     void planEmulatedE3M0ToHF(CopyInstruction &i);
+    void planEmulatedHFToE3M0(CopyInstruction &i);
     void planEmulatedF4E2M1ToHF(CopyInstruction &i);
     void planEmulatedHFToF4E2M1(CopyInstruction &i);
     void planInt4Upconversion(CopyInstruction &i);
diff --git a/src/gpu/intel/jit/gemm/generator/pieces/ngen_object_helpers.hpp b/src/gpu/intel/jit/gemm/generator/pieces/ngen_object_helpers.hpp
index 5ce5018d994..fdb3575ea4f 100644
--- a/src/gpu/intel/jit/gemm/generator/pieces/ngen_object_helpers.hpp
+++ b/src/gpu/intel/jit/gemm/generator/pieces/ngen_object_helpers.hpp
@@ -31,7 +31,7 @@ static inline bool isW(ngen::DataType dt) { return one_of(dt, ngen::DataType::uw
 static inline bool isD(ngen::DataType dt) { return one_of(dt, ngen::DataType::ud, ngen::DataType::d); }
 static inline bool isQ(ngen::DataType dt) { return one_of(dt, ngen::DataType::uq, ngen::DataType::q); }
 static inline bool isFP8(ngen::DataType dt) { return (dt == Type::ngen_f8_e8m0()) || one_of(dt, ngen::DataType::bf8, ngen::DataType::hf8); }
-static inline bool isFP4(ngen::DataType dt) { return dt == Type::ngen_f4_e2m1(); }
+static inline bool isFP4(ngen::DataType dt) { return (dt == Type::ngen_f4_e2m1() || dt == Type::ngen_f4_e3m0()); }
 static inline bool isInt4(ngen::DataType dt) { return one_of(dt, ngen::DataType::u4, ngen::DataType::s4);}
 static inline bool is4Bit(ngen::DataType dt) { return isFP4(dt) || isInt4(dt);}
 
diff --git a/src/gpu/intel/jit/gemm/selector/db/kernel.db b/src/gpu/intel/jit/gemm/selector/db/kernel.db
index 3ab81c5c667..ecb06f3fa64 100644
--- a/src/gpu/intel/jit/gemm/selector/db/kernel.db
+++ b/src/gpu/intel/jit/gemm/selector/db/kernel.db
@@ -595,13 +595,13 @@ auto _CATALOG_ = kcatalog::toArray({
 {{'F', "gemm", {"D", "D", "D"}, {"T", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, ""}, "aS8x2+S16@24 aS16+S16@32 aB wg 4x2x4 kr kc8 nse hi pt sr br bk0 sm sn grf256 kv afb sb32", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 2}, {131072, 131072, 16777216}, {8192, 8192, 0}, {8, 8, 16}, {4, 2, 4}, 1, (WGType) 1, 413, 0, 4096, {8, 8, 8}, {true, true, true}}, {'E', 17, {1.2692e+06, -138155, -69946.2, 247166, 2.48218e+06, 0, 2.22535, 2.9076, -0.226323, 2.36118, 0.179437, 0.0955498, 0.0965902, 0.986833, 1.48781, 1.14074, 6.5259e-12}}},
 {{'F', "gemm", {"D", "D", "D"}, {"T", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, ""}, "aS16+S8@32 aS8+S1,8@32 aB wg 16x1x2 kr kc8 nse li nmk pt sr br bk0 sm sn grf256 sb32 kv afb", {16, (LoopType) 255, 256, {(LoopType) 225, (LoopType) 255, (LoopType) 2}, {131072, 262144, 16777216}, {8192, 8192, 0}, {8, 16, 16}, {16, 1, 2}, 1, (WGType) 1, 413, 0, 16384, {8, 8, 8}, {true, true, true}}, {'E', 17, {1.08982e+06, -365235, -57570.3, 563017, 2.44122e+06, 0, 1.97404, 6.45479, 0.729323, 2.0721, 0.109576, 0.0188555, 0.121551, 0.995776, 1.6362, 1.09808, 8.84348e-12}}},
 {{'F', "gemm", {"D", "D", "D"}, {"T", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, ""}, "aS8+S1,8@16 aB8+B8@16 aS nse wg 8x4 bo pt kc8 sm sb256 grf256 bk0 sr", {16, (LoopType) 255, 256, {(LoopType) 192, (LoopType) 255, (LoopType) 255}, {524288, 524288, 16777216}, {8192, 8192, 16777216}, {32, 32, 8}, {8, 4, 1}, 1, (WGType) 1, 257, 0, 0, {8, 8, 8}, {true, true, true}}, {'E', 17, {863507, 843091, 0, 0, 0, 0, 2.99578, 2.42156, 4.42824, 4.74906, 0.0650645, 0.0650645, 0, 0.993213, 1.59693, 1.16972, 1.1178e-11}}},
-{{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "#I"}, "aB16+m32@32 aB32 aB wg 2x8 af vav li nmk pt sr br ca3 bk0 sys kv dm afb l4", {16, (LoopType) 255, 128, {(LoopType) 225, (LoopType) 255, (LoopType) 255}, {1048576, 32768, 16777216}, {1048576, 32768, 32}, {64, 2, 32}, {2, 8, 1}, 1, (WGType) 1, 441, 24576, 0, {4, 2, 4}, {true, true, true}}, {'E', 17, {1.32162e+06, 161954, 0, 0, 2.32817e+06, 0, 0.71806, 4.15517, 0.786689, 1.40778, 0.0341164, 0.0131941, 0.0256486, 0.947188, 1.39057, 0.987284, 5.0128e-12}}},
+{{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "#I"}, "aB16+m32@32 aB32 aB wg 2x8 af vav li nmk pt sr br ca3 bk0 sys kv dm afb l4", {16, (LoopType) 255, 128, {(LoopType) 225, (LoopType) 255, (LoopType) 255}, {1048576, 32768, 16777216}, {1048576, 32768, 32}, {32, 2, 32}, {2, 8, 1}, 1, (WGType) 1, 441, 24576, 0, {4, 2, 4}, {true, true, true}}, {'E', 17, {1.32162e+06, 161954, 0, 0, 2.32817e+06, 0, 0.71806, 4.15517, 0.786689, 1.40778, 0.0341164, 0.0131941, 0.0256486, 0.947188, 1.39057, 0.987284, 5.0128e-12}}},
 {{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABI"}, "av64+m64@64 am32x2+m64@32 aB wg 4x8 xaf vav hi pt sr br sb64 bk0 sn grf256 sys", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {262144, 131072, 16777216}, {262144, 131072, 16777216}, {32, 32, 64}, {4, 8, 1}, 1, (WGType) 1, 257, 0, 0, {16, 16, 4}, {true, true, true}}, {'E', 17, {888453, 388680, 0, 0, 0, 0, 1.36981, 1.36706, 1.57389, 2.50026, 0.0230892, 0.0230892, 0, 0.779666, 1.13216, 0.970824, 1.11098e-12}}},
 {{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABI"}, "av16+m64@64 am32+m64@64 aB wg 4x8 af vav hi pt sr br sb256 bk0 sn grf256 sys", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {524288, 262144, 16777216}, {524288, 262144, 16777216}, {32, 16, 32}, {4, 8, 1}, 1, (WGType) 1, 257, 0, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {872440, 433979, 0, 0, 0, 0, 0.692755, 0.929392, 0.682568, 1.28977, 0.00829318, 0.00829318, 0, 0.933146, 1.44966, 1.06633, 2.17433e-12}}},
 {{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABI"}, "av16+m64@64 am16 aB wg 8x4 cb4x2 ks32 xaf vav hi pt sr br bk0 sn grf256 sys", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {524288, 524288, 16777216}, {524288, 524288, 16777216}, {32, 32, 32}, {8, 4, 1}, 1, (WGType) 1, 257, 32768, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {1.00706e+06, 522382, 0, 0, 0, 0, 0.725659, 1.44632, 0.970408, 1.74134, 0.0067111, 0.0067111, 0, 0.90349, 1.42986, 1.13348, 2.91269e-12}}},
 {{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABIps"}, "av32+m16@64 am32+m32@72 aB wg 8x4 xaf vav hi pt sr br sb64 bk0 sn grf256 sys kv afb", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {524288, 786432, 16777216}, {524288, 786432, 64}, {32, 16, 32}, {8, 4, 1}, 1, (WGType) 1, 441, 0, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {876646, 564122, 0, 0, 6.5151e+06, 7.83974e+06, 0.629669, 0.87362, 0.885543, 1.48097, 0.00440774, 0.00440774, 0, 1, 1.66234, 1.24996, 2.85794e-12}}},
 {{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABI"}, "av16+m16@32 am16x2 aB wg 4x4x2 kr cb4 ks16 xaf st vav hi pt sr br bk0 sn grf256 sys", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 2}, {1048576, 524288, 16777216}, {1048576, 524288, 16777216}, {64, 32, 16}, {4, 4, 2}, 1, (WGType) 1, 261, 16384, 65536, {8, 8, 4}, {true, true, true}}, {'E', 17, {1.02495e+06, 13797.9, 15430.1, 758509, 0, 0, 0.535333, 1.20812, 0.912657, 1.84068, 0.00529983, 0.00529983, 0, 1, 1.60581, 1.15873, 3.51036e-12}}},
-{{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABIqps"}, "av16+m32@72 am32+m32@64 aB wg 4x8 xaf vav hi pt sr br sb64 bk0 sn grf256 sys kv afb", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {1048576, 655360, 16777216}, {1048576, 655360, 32}, {64, 24, 32}, {4, 8, 1}, 1, (WGType) 1, 441, 0, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {903365, 697556, 0, 0, 8.2903e+06, 1.21651e+07, 0.724506, 0.722081, 0.92287, 1.55416, 0.00402055, 0.00402055, 0, 0.997691, 1.6726, 1.18622, 5.18793e-12}}},
+{{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABIqps"}, "av16+m32@72 am32+m32@64 aB wg 4x8 xaf vav hi pt sr br sb64 bk0 sn grf256 sys kv afb", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {1048576, 655360, 16777216}, {1048576, 655360, 32}, {32, 16, 32}, {4, 8, 1}, 1, (WGType) 1, 441, 0, 0, {8, 8, 4}, {true, true, true}}, {'E', 17, {903365, 697556, 0, 0, 8.2903e+06, 1.21651e+07, 0.724506, 0.722081, 0.92287, 1.55416, 0.00402055, 0.00402055, 0, 0.997691, 1.6726, 1.18622, 5.18793e-12}}},
 {{'F', "gemm", {"E", "E", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {8, 8, 1}, "ABI"}, "av16+m32@40 am32+m32@32 aB wg 2x8x2 kr xaf st vav hi pt sr br sb64 bk0 sn grf256 sys", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 2}, {1048576, 131072, 16777216}, {1048576, 131072, 16777216}, {64, 8, 32}, {2, 8, 2}, 1, (WGType) 1, 261, 0, 32768, {8, 8, 4}, {true, true, true}}, {'E', 17, {1.06324e+06, 220443, 364.907, 276934, 0, 0, 0.524524, 1.25881, 0.793843, 2.21167, 0.00974309, 0.00974309, 0, 0.984682, 1.55809, 1.03396, 4.08729e-12}}},
 {{'F', "gemm", {"F", "B", "S"}, {"N", "N", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "#I"}, "aB16x2 aB16x2 aB wg 4x8 cab4 ks64 af vav hi pt sr br bk0 sn nb 4x8 dm grf256 sys l4", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 255}, {524288, 131072, 16777216}, {524288, 131072, 16777216}, {32, 8, 64}, {4, 8, 1}, 1, (WGType) 1, 257, 98304, 0, {2, 2, 4}, {true, true, true}}, {'E', 17, {1.07581e+06, 764320, 0, 0, 0, 0, 0.804535, 1.46469, 0.96438, 2.27185, 0.0120677, 0.0120677, 0, 1, 1.38109, 0.955498, 2.48341e-12}}},
 {{'F', "gemm", {"F", "B", "S"}, {"N", "T", "N"}}, {-1, -1, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {-1, -1, -1}, {1, 1, 1}, "#I"}, "aB16 at16 aB wg 1x2x16 kr cab3x2 ks16 af vav hi pt sr br bk0 grf256 kv afb sys sn l4 l2d", {16, (LoopType) 255, 256, {(LoopType) 208, (LoopType) 255, (LoopType) 2}, {524288, 262144, 16777216}, {524288, 262144, 32}, {32, 16, 16}, {1, 2, 16}, 1, (WGType) 1, 445, 6144, 6144, {2, 2, 4}, {true, true, true}}, {'E', 17, {1.44116e+06, -85531, -20925.3, 122459, 4.34995e+06, 2.28557e+06, 0.844362, 0.875124, 0.584811, 1.33561, 0.0318181, 0.0286306, 0.00945064, 0.972737, 1.204, 0.897885, 9.01622e-12}}},
diff --git a/src/gpu/intel/ocl/ref_matmul.hpp b/src/gpu/intel/ocl/ref_matmul.hpp
index 1a883c1ebf4..d73f4d2dcb3 100644
--- a/src/gpu/intel/ocl/ref_matmul.hpp
+++ b/src/gpu/intel/ocl/ref_matmul.hpp
@@ -89,7 +89,8 @@ struct ref_matmul_t : public gpu_primitive_t {
             const bool is_f4
                     = ((utils::one_of(src_dt_, f4_e2m1, f4_e3m0)
                                || utils::everyone_is(wei_dt_, f4_e2m1, f4_e3m0))
-                            && utils::one_of(dst_dt_, f32, bf16, f16, src_dt_));
+                            && utils::one_of(dst_dt_, f32, bf16, f16, f4_e3m0,
+                                    f4_e2m1, src_dt_));
             const bool is_int8 = utils::one_of(src_dt_, u8, s8)
                     && utils::one_of(wei_dt_, u8, s8, u4, s4)
                     && utils::one_of(dst_dt_, f32, s8, u8, s32, f16);
diff --git a/tests/benchdnn/inputs/matmul/test_matmul_fp4 b/tests/benchdnn/inputs/matmul/test_matmul_fp4
index 3639dcde2d8..d958bcff592 100644
--- a/tests/benchdnn/inputs/matmul/test_matmul_fp4
+++ b/tests/benchdnn/inputs/matmul/test_matmul_fp4
@@ -8,7 +8,7 @@
 --attr-post-ops=
 
 ## plain fp4
---dt=f4_e2m1,f4_e2m1:f4_e2m1:f32,f4_e2m1:f4_e2m1:bf16,f4_e2m1:f4_e3m0:f4_e2m1
+--dt=f4_e2m1,f4_e3m0,f4_e2m1:f4_e2m1:f32,f4_e2m1:f4_e2m1:bf16,f4_e2m1:f4_e3m0:f4_e2m1,f4_e2m1:f4_e2m1:f4_e3m0
 --attr-scales=
 --batch=shapes_4bit
 
diff --git a/third_party/ngen/ngen.hpp b/third_party/ngen/ngen.hpp
index e46fce73523..21a32d3302f 100644
--- a/third_party/ngen/ngen.hpp
+++ b/third_party/ngen/ngen.hpp
@@ -1991,7 +1991,7 @@ BinaryCodeGenerator<hw>::opX(Opcode op, DataType defaultType, const InstructionM
     if (forceWE)
         emod |= NoMask;
 
-    int ewidth = getExecWidth({defaultType, dst.getType(), src0.getType(), src1.getType()});
+    int ewidth = getExecWidth({defaultType, dst.getType(), src0.getType(), src1.isNull() ? src0.getType()  : src1.getType()});
     dst.fixup(hw,  emod.getExecSize(), ewidth, defaultType, -1, 2);
     src0.fixup(hw, emod.getExecSize(), ewidth, defaultType, 0, 2);
     src1.fixup(hw, emod.getExecSize(), ewidth, defaultType, 1, 2);
@@ -2676,6 +2676,9 @@ template <HW hw_>
 typename std::enable_if<hwLT(hw_, HW::Gen12LP)>::type
 BinaryCodeGenerator<hw>::opJmpi(Opcode op, const InstructionModifier &mod, const RegData &dst, RegData src0, uint32_t jip, SourceLocation loc)
 {
+#ifdef NGEN_SAFE
+        if (mod.getExecSize() != 1) throw invalid_modifiers_exception();
+#endif
     debugLine.add(rootStream.length(), loc);
 
     Instruction8 i{};
diff --git a/third_party/ngen/ngen_core.hpp b/third_party/ngen/ngen_core.hpp
index 59c08661c3c..e374227eabc 100644
--- a/third_party/ngen/ngen_core.hpp
+++ b/third_party/ngen/ngen_core.hpp
@@ -1216,6 +1216,7 @@ class ExtendedReg {
     constexpr bool isValid()        const { return !base.isInvalid(); }
     constexpr bool isScalar()       const { return base.isScalar(); }
     constexpr bool isARF()          const { return base.isARF(); }
+    constexpr bool isNull()          const { return base.isNull(); }
 
     constexpr14 RegData &getBase()        { return base; }
     constexpr RegData getBase()     const { return base; }


Reviews:
user: login: dzarukin
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-02-27T04:38:23Z
user: login: petercad
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-27T23:23:16Z
user: login: petercad
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-27T23:27:56Z
user: login: petercad
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-27T23:37:16Z
user: login: kealan-barbieri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T01:24:20Z
user: login: kealan-barbieri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-02-28T01:25:04Z
user: login: petercad
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-05T17:32:01Z
user: login: kealan-barbieri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-07T19:43:46Z
user: login: atkassen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T18:06:16Z
user: login: kealan-barbieri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T18:50:09Z
user: login: kealan-barbieri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-10T18:52:31Z
user: login: kealan-barbieri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T18:29:07Z
user: login: atkassen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-11T22:09:14Z
user: login: kealan-barbieri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T16:51:00Z
user: login: atkassen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T17:22:55Z
user: login: kealan-barbieri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-12T17:55:28Z
user: login: atkassen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T16:05:04Z
user: login: kealan-barbieri
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T16:22:47Z
user: login: atkassen
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T16:29:49Z
user: login: atkassen
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-17T17:03:21Z

Review comments:
diff_hunk: @@ -1479,6 +1488,309 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+void CopyPlan::allocSharedF4E3M0() {
+    if(sharedAlloced) return;
+    sharedIdx = newTemp(DataType::uw, 1, 1, 0, 0);
+    sharedIdxFlag = newFlag(1);
+    sharedTmp = newTemp(DataType::hf, 1, 1, 1, 1);
+
+}
+
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
+    // mov                 idx:uw       0:uw
+    // mov                 bits:uw      0:uw
+    // mov                 x<1>:uw      x<2>:uw
+    // mov                 tmin:uw      x:uw
+    // mov                 tmp<0>:hf    0x3400:hf
+    // Label loopTop
+    // add                 idx:uw       idx:uw          0x1:uw
+    // add                 tdiff:hf     tmp<0>:hf       -x:hf
+    // (f0)and (ZE)        null         idx<0>:uw       0x1:uw
+    // jmpi (f0) labelEven
+    // (f1)cmp (LT)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // jmp condEnd
+    // Label labelEven
+    // (f1)cmp (LE)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // Label condEnd
+    // mov (f1)            bits:uw      idx<0>:uw
+    // mov (f1)            tmin:uw      tdiff:uw
+    // (f0)cmp (LT)        null:uw      idx:uw          0x0007:uw
+    // mul (f1)            tmp:hf       tmp:hf          0x4000:hf
+    // jmpi (f0) loopTop
+    // (f1)and             null:uw      x:uw            0x1000:uw
+    // or (f1)             bits:uw      bits:uw         0x8:uw
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: petercad
body: (Random location) I might have missed it, but I didn't see any special handling to avoid reordering instructions around control flow. Near the end of the copy planner there's a pass to reorder instructions based on their phase, so as to separate dependent instructions. `splitMultiple` will give each instruction its own phase, so if you have e.g. multiple hf -> e3m0 moves in the input, the two instruction sequences created by this routine will be interleaved.
created_at: 2025-02-27T23:23:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1488,309 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+void CopyPlan::allocSharedF4E3M0() {
+    if(sharedAlloced) return;
+    sharedIdx = newTemp(DataType::uw, 1, 1, 0, 0);
+    sharedIdxFlag = newFlag(1);
+    sharedTmp = newTemp(DataType::hf, 1, 1, 1, 1);
+
+}
+
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
+    // mov                 idx:uw       0:uw
+    // mov                 bits:uw      0:uw
+    // mov                 x<1>:uw      x<2>:uw
+    // mov                 tmin:uw      x:uw
+    // mov                 tmp<0>:hf    0x3400:hf
+    // Label loopTop
+    // add                 idx:uw       idx:uw          0x1:uw
+    // add                 tdiff:hf     tmp<0>:hf       -x:hf
+    // (f0)and (ZE)        null         idx<0>:uw       0x1:uw
+    // jmpi (f0) labelEven
+    // (f1)cmp (LT)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // jmp condEnd
+    // Label labelEven
+    // (f1)cmp (LE)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // Label condEnd
+    // mov (f1)            bits:uw      idx<0>:uw
+    // mov (f1)            tmin:uw      tdiff:uw
+    // (f0)cmp (LT)        null:uw      idx:uw          0x0007:uw
+    // mul (f1)            tmp:hf       tmp:hf          0x4000:hf
+    // jmpi (f0) loopTop
+    // (f1)and             null:uw      x:uw            0x1000:uw
+    // or (f1)             bits:uw      bits:uw         0x8:uw
+    // 
+    // /* Pack into bytes */
+
+    if (i.src0.neg || i.sat || i.hasCMod()) stub("Unsupported modifier");
+
+    allocSharedF4E3M0();
+
+    auto ie = splitMultiple<27>(i);
+
+    auto yOrig = i.dst, y = yOrig;
+    auto xOrig = i.src0, x = xOrig;
+
+    bool tempX = (x.stride > 1);
+
+    auto bits = newTemp(DataType::uw, i.simd, 1, 0, 0);
+    auto tmin = newTemp(DataType::hf, i.simd, 1, 0, 0);
+    auto ldiff = newTemp(DataType::hf, i.simd, 1, 0, 0);
+
+    auto selFlag = newFlag(i.simd);
+    int simd = i.simd;
+    ngen::Label * loopTop = newLabel();
+    ngen::Label * labelEven = newLabel();
+    ngen::Label * condEnd = newLabel();
+    int idx = 0;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = 1;
+    ie[idx]->cFlow = true;
+    ie[idx]->dst = sharedIdx;
+    ie[idx]->src0 = Immediate::uw(0x0);
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = simd;
+    ie[idx]->dst = bits;
+    ie[idx]->src0 = Immediate::uw(0x0);
+    ++idx;
+
+    if(tempX){
+        ie[idx]->op = Opcode::mov;
+        ie[idx]->dst = x;
+        ie[idx]->dst.stride = 1;
+        ie[idx]->src0 = x;
+    }else{
+        ie[idx]->invalidate();
+    }
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->dst = tmin;
+    ie[idx]->src0 = x;
+    ie[idx]->src0.stride = 1;
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = 1;
+    ie[idx]->cFlow = true;
+    ie[idx]->dst = sharedTmp;
+    ie[idx]->src0 = Immediate::hf(0x3400);
+    ++idx;
+
+    // Loop Begin
+    ie[idx]->op = Opcode::nop;
+    ie[idx]->label = true;
+    ie[idx]->label_ = loopTop;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::add;
+    ie[idx]->dst = sharedIdx;
+    ie[idx]->src0 = sharedIdx;
+    ie[idx]->src1 = Immediate::uw(0x1);
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::add;
+    ie[idx]->dst = ldiff;
+    ie[idx]->src0 = sharedTmp;
+    ie[idx]->src0.stride = 0;
+    ie[idx]->src1 = -x;
+    ie[idx]->src1.stride = 1;
+    ie[idx]->src1.abs = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::and_;
+    ie[idx]->cmod = ConditionModifier::ze;
+    ie[idx]->flag = sharedIdxFlag;
+    ie[idx]->cFlow = true;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = sharedIdx;
+    ie[idx]->src1 = Immediate::uw(0x1);
+    ++idx;
+
+    ie[idx]->op = Opcode::jmpi;
+    ie[idx]->simd = 1;
+    ie[idx]->flag = sharedIdxFlag;
+    ie[idx]->label_ = labelEven;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    // Condition Odd.
+    ie[idx]->op = Opcode::cmp;
+    ie[idx]->cmod = ConditionModifier::lt;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::hf;
+    ie[idx]->src0 = ldiff;
+    ie[idx]->src0.abs = true;
+    ie[idx]->src1 = tmin;
+    ie[idx]->src1.abs = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::jmpi;
+    ie[idx]->simd = 1;
+    ie[idx]->label_ = condEnd;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    // Condition Even.
+    ie[idx]->op = Opcode::nop;
+    ie[idx]->label = true;
+    ie[idx]->label_ = labelEven;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::cmp;
+    ie[idx]->cmod = ConditionModifier::le;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::hf;
+    ie[idx]->src0 = ldiff;
+    ie[idx]->src0.abs = true;
+    ie[idx]->src1 = tmin;
+    ie[idx]->src1.abs = true;
+    ++idx;
+
+    // Condition End
+    ie[idx]->op = Opcode::nop;
+    ie[idx]->label = true;
+    ie[idx]->label_ = condEnd;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = bits;
+    ie[idx]->src0 = sharedIdx;
+    ie[idx]->src0.stride = 0;
+    ++idx;
+
+    // Update minimum diff, results.
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = tmin;
+    ie[idx]->src0 = ldiff;
+    ++idx;
+
+    ie[idx]->op = Opcode::cmp;
+    ie[idx]->flag = sharedIdxFlag;
+    ie[idx]->cFlow = true;
+    ie[idx]->cmod = ConditionModifier::lt;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = sharedIdx;
+    ie[idx]->src1 = Immediate::uw(0x7); 
+    ++idx;
+
+    ie[idx]->op = Opcode::mul;
+    ie[idx]->simd = 1;
+    ie[idx]->cFlow = true;
+    ie[idx]->dst = sharedTmp;
+    ie[idx]->src0 = sharedTmp;
+    ie[idx]->src1 = Immediate::hf(0x4000);
+    ++idx;
+
+    // Loop Iter.
+    ie[idx]->op = Opcode::jmpi;
+    ie[idx]->simd = 1;
+    ie[idx]->flag = sharedIdxFlag;
+    ie[idx]->label_ = loopTop;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    // Apply sign.
+    ie[idx]->op = Opcode::and_;
+    ie[idx]->flag = selFlag;
+    ie[idx]->cmod = ConditionModifier::nz;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = x;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src1 = Immediate::uw(0x8000); 
+    ++idx;
+
+    ie[idx]->op = Opcode::or_;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = bits;
+    ie[idx]->src0 = bits;
+    ie[idx]->src1 = Immediate::uw(0x8); 
+    ++idx;
+
+    // Pack.
+    ie[idx]->op = Opcode::mov;
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: petercad
body: It would be great to factor out the int4 packing operation from this routine and the hf->e2m1 sequence into its own conversion method (which basically implements an int8 -> int4 copy). Then, as a side benefit, you can do arbitrary int4->int4 by copying through int8, which also implements e3m0->e3m0 or e2m1->e2m1.
created_at: 2025-02-27T23:27:56Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1488,309 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+void CopyPlan::allocSharedF4E3M0() {
+    if(sharedAlloced) return;
+    sharedIdx = newTemp(DataType::uw, 1, 1, 0, 0);
+    sharedIdxFlag = newFlag(1);
+    sharedTmp = newTemp(DataType::hf, 1, 1, 1, 1);
+
+}
+
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
+    // mov                 idx:uw       0:uw
+    // mov                 bits:uw      0:uw
+    // mov                 x<1>:uw      x<2>:uw
+    // mov                 tmin:uw      x:uw
+    // mov                 tmp<0>:hf    0x3400:hf
+    // Label loopTop
+    // add                 idx:uw       idx:uw          0x1:uw
+    // add                 tdiff:hf     tmp<0>:hf       -x:hf
+    // (f0)and (ZE)        null         idx<0>:uw       0x1:uw
+    // jmpi (f0) labelEven
+    // (f1)cmp (LT)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // jmp condEnd
+    // Label labelEven
+    // (f1)cmp (LE)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // Label condEnd
+    // mov (f1)            bits:uw      idx<0>:uw
+    // mov (f1)            tmin:uw      tdiff:uw
+    // (f0)cmp (LT)        null:uw      idx:uw          0x0007:uw
+    // mul (f1)            tmp:hf       tmp:hf          0x4000:hf
+    // jmpi (f0) loopTop
+    // (f1)and             null:uw      x:uw            0x1000:uw
+    // or (f1)             bits:uw      bits:uw         0x8:uw
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: petercad
body: That said, I think it's better to avoid control flow entirely. Maybe something like:
* extract exponent field from src
* subtract bias difference + clamp to zero
* add MSB of mantissa to exponent (= round to nearest or even)
* exponent = min(exponent, 7)
* combine exponent and sign
created_at: 2025-02-27T23:37:16Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1488,309 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+void CopyPlan::allocSharedF4E3M0() {
+    if(sharedAlloced) return;
+    sharedIdx = newTemp(DataType::uw, 1, 1, 0, 0);
+    sharedIdxFlag = newFlag(1);
+    sharedTmp = newTemp(DataType::hf, 1, 1, 1, 1);
+
+}
+
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
+    // mov                 idx:uw       0:uw
+    // mov                 bits:uw      0:uw
+    // mov                 x<1>:uw      x<2>:uw
+    // mov                 tmin:uw      x:uw
+    // mov                 tmp<0>:hf    0x3400:hf
+    // Label loopTop
+    // add                 idx:uw       idx:uw          0x1:uw
+    // add                 tdiff:hf     tmp<0>:hf       -x:hf
+    // (f0)and (ZE)        null         idx<0>:uw       0x1:uw
+    // jmpi (f0) labelEven
+    // (f1)cmp (LT)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // jmp condEnd
+    // Label labelEven
+    // (f1)cmp (LE)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // Label condEnd
+    // mov (f1)            bits:uw      idx<0>:uw
+    // mov (f1)            tmin:uw      tdiff:uw
+    // (f0)cmp (LT)        null:uw      idx:uw          0x0007:uw
+    // mul (f1)            tmp:hf       tmp:hf          0x4000:hf
+    // jmpi (f0) loopTop
+    // (f1)and             null:uw      x:uw            0x1000:uw
+    // or (f1)             bits:uw      bits:uw         0x8:uw
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: kealan-barbieri
body: I have some handling for control flow instructions in `materializeTemps`, that makes sure theres only one set of control flow instructions within each cnum interval. It appears to handle the cases you are talking about. I will look into implementing the non-iterative conversion method.
created_at: 2025-02-28T01:24:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1488,309 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+void CopyPlan::allocSharedF4E3M0() {
+    if(sharedAlloced) return;
+    sharedIdx = newTemp(DataType::uw, 1, 1, 0, 0);
+    sharedIdxFlag = newFlag(1);
+    sharedTmp = newTemp(DataType::hf, 1, 1, 1, 1);
+
+}
+
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
+    // mov                 idx:uw       0:uw
+    // mov                 bits:uw      0:uw
+    // mov                 x<1>:uw      x<2>:uw
+    // mov                 tmin:uw      x:uw
+    // mov                 tmp<0>:hf    0x3400:hf
+    // Label loopTop
+    // add                 idx:uw       idx:uw          0x1:uw
+    // add                 tdiff:hf     tmp<0>:hf       -x:hf
+    // (f0)and (ZE)        null         idx<0>:uw       0x1:uw
+    // jmpi (f0) labelEven
+    // (f1)cmp (LT)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // jmp condEnd
+    // Label labelEven
+    // (f1)cmp (LE)        null:hf      abs(tmin:hf)    abs(tdiff:hf)
+    // Label condEnd
+    // mov (f1)            bits:uw      idx<0>:uw
+    // mov (f1)            tmin:uw      tdiff:uw
+    // (f0)cmp (LT)        null:uw      idx:uw          0x0007:uw
+    // mul (f1)            tmp:hf       tmp:hf          0x4000:hf
+    // jmpi (f0) loopTop
+    // (f1)and             null:uw      x:uw            0x1000:uw
+    // or (f1)             bits:uw      bits:uw         0x8:uw
+    // 
+    // /* Pack into bytes */
+
+    if (i.src0.neg || i.sat || i.hasCMod()) stub("Unsupported modifier");
+
+    allocSharedF4E3M0();
+
+    auto ie = splitMultiple<27>(i);
+
+    auto yOrig = i.dst, y = yOrig;
+    auto xOrig = i.src0, x = xOrig;
+
+    bool tempX = (x.stride > 1);
+
+    auto bits = newTemp(DataType::uw, i.simd, 1, 0, 0);
+    auto tmin = newTemp(DataType::hf, i.simd, 1, 0, 0);
+    auto ldiff = newTemp(DataType::hf, i.simd, 1, 0, 0);
+
+    auto selFlag = newFlag(i.simd);
+    int simd = i.simd;
+    ngen::Label * loopTop = newLabel();
+    ngen::Label * labelEven = newLabel();
+    ngen::Label * condEnd = newLabel();
+    int idx = 0;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = 1;
+    ie[idx]->cFlow = true;
+    ie[idx]->dst = sharedIdx;
+    ie[idx]->src0 = Immediate::uw(0x0);
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = simd;
+    ie[idx]->dst = bits;
+    ie[idx]->src0 = Immediate::uw(0x0);
+    ++idx;
+
+    if(tempX){
+        ie[idx]->op = Opcode::mov;
+        ie[idx]->dst = x;
+        ie[idx]->dst.stride = 1;
+        ie[idx]->src0 = x;
+    }else{
+        ie[idx]->invalidate();
+    }
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->dst = tmin;
+    ie[idx]->src0 = x;
+    ie[idx]->src0.stride = 1;
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->simd = 1;
+    ie[idx]->cFlow = true;
+    ie[idx]->dst = sharedTmp;
+    ie[idx]->src0 = Immediate::hf(0x3400);
+    ++idx;
+
+    // Loop Begin
+    ie[idx]->op = Opcode::nop;
+    ie[idx]->label = true;
+    ie[idx]->label_ = loopTop;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::add;
+    ie[idx]->dst = sharedIdx;
+    ie[idx]->src0 = sharedIdx;
+    ie[idx]->src1 = Immediate::uw(0x1);
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::add;
+    ie[idx]->dst = ldiff;
+    ie[idx]->src0 = sharedTmp;
+    ie[idx]->src0.stride = 0;
+    ie[idx]->src1 = -x;
+    ie[idx]->src1.stride = 1;
+    ie[idx]->src1.abs = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::and_;
+    ie[idx]->cmod = ConditionModifier::ze;
+    ie[idx]->flag = sharedIdxFlag;
+    ie[idx]->cFlow = true;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = sharedIdx;
+    ie[idx]->src1 = Immediate::uw(0x1);
+    ++idx;
+
+    ie[idx]->op = Opcode::jmpi;
+    ie[idx]->simd = 1;
+    ie[idx]->flag = sharedIdxFlag;
+    ie[idx]->label_ = labelEven;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    // Condition Odd.
+    ie[idx]->op = Opcode::cmp;
+    ie[idx]->cmod = ConditionModifier::lt;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::hf;
+    ie[idx]->src0 = ldiff;
+    ie[idx]->src0.abs = true;
+    ie[idx]->src1 = tmin;
+    ie[idx]->src1.abs = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::jmpi;
+    ie[idx]->simd = 1;
+    ie[idx]->label_ = condEnd;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    // Condition Even.
+    ie[idx]->op = Opcode::nop;
+    ie[idx]->label = true;
+    ie[idx]->label_ = labelEven;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::cmp;
+    ie[idx]->cmod = ConditionModifier::le;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::hf;
+    ie[idx]->src0 = ldiff;
+    ie[idx]->src0.abs = true;
+    ie[idx]->src1 = tmin;
+    ie[idx]->src1.abs = true;
+    ++idx;
+
+    // Condition End
+    ie[idx]->op = Opcode::nop;
+    ie[idx]->label = true;
+    ie[idx]->label_ = condEnd;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = bits;
+    ie[idx]->src0 = sharedIdx;
+    ie[idx]->src0.stride = 0;
+    ++idx;
+
+    // Update minimum diff, results.
+    ie[idx]->op = Opcode::mov;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = tmin;
+    ie[idx]->src0 = ldiff;
+    ++idx;
+
+    ie[idx]->op = Opcode::cmp;
+    ie[idx]->flag = sharedIdxFlag;
+    ie[idx]->cFlow = true;
+    ie[idx]->cmod = ConditionModifier::lt;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = sharedIdx;
+    ie[idx]->src1 = Immediate::uw(0x7); 
+    ++idx;
+
+    ie[idx]->op = Opcode::mul;
+    ie[idx]->simd = 1;
+    ie[idx]->cFlow = true;
+    ie[idx]->dst = sharedTmp;
+    ie[idx]->src0 = sharedTmp;
+    ie[idx]->src1 = Immediate::hf(0x4000);
+    ++idx;
+
+    // Loop Iter.
+    ie[idx]->op = Opcode::jmpi;
+    ie[idx]->simd = 1;
+    ie[idx]->flag = sharedIdxFlag;
+    ie[idx]->label_ = loopTop;
+    ie[idx]->cFlow = true;
+    ++idx;
+
+    // Apply sign.
+    ie[idx]->op = Opcode::and_;
+    ie[idx]->flag = selFlag;
+    ie[idx]->cmod = ConditionModifier::nz;
+    ie[idx]->dst = CopyOperand();
+    ie[idx]->dst.type = DataType::uw;
+    ie[idx]->src0 = x;
+    ie[idx]->src0.stride = 1;
+    ie[idx]->src0.type = DataType::uw;
+    ie[idx]->src1 = Immediate::uw(0x8000); 
+    ++idx;
+
+    ie[idx]->op = Opcode::or_;
+    ie[idx]->flag = selFlag;
+    ie[idx]->dst = bits;
+    ie[idx]->src0 = bits;
+    ie[idx]->src1 = Immediate::uw(0x8); 
+    ++idx;
+
+    // Pack.
+    ie[idx]->op = Opcode::mov;
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: kealan-barbieri
body: Makes sense, will add this.
created_at: 2025-02-28T01:25:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -671,12 +671,18 @@ void CopyPlan::planTypeConversions()
         } else if (st == Type::ngen_f4_e3m0() && dt == DataType::hf) {
             planEmulatedE3M0ToHF(i);
             rerun = true;
-        } else if (dt == Type::ngen_f4_e2m1()) {
+        } else if (dt == Type::ngen_f4_e3m0() && st == DataType::hf) {
+            planEmulatedHFToE3M0(i);
+            rerun = true;
+        } else if (isFP4(dt)) {
             copyThrough(i, DataType::hf);
             rerun = true;
-        } else if (st == Type::ngen_f4_e2m1()) {
+        } else if (isFP4(st)) {
             copyThrough(i, DataType::hf);
             rerun = true;
+        } else if (one_of(st, DataType::ub, DataType::uw) && dt == DataType::u4) {
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: petercad
body: Sign doesn't matter here, right?
```suggestion
        } else if ((isB(st) || isW(st)) && isInt4(dt)) {
```
created_at: 2025-03-05T17:32:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -671,12 +671,18 @@ void CopyPlan::planTypeConversions()
         } else if (st == Type::ngen_f4_e3m0() && dt == DataType::hf) {
             planEmulatedE3M0ToHF(i);
             rerun = true;
-        } else if (dt == Type::ngen_f4_e2m1()) {
+        } else if (dt == Type::ngen_f4_e3m0() && st == DataType::hf) {
+            planEmulatedHFToE3M0(i);
+            rerun = true;
+        } else if (isFP4(dt)) {
             copyThrough(i, DataType::hf);
             rerun = true;
-        } else if (st == Type::ngen_f4_e2m1()) {
+        } else if (isFP4(st)) {
             copyThrough(i, DataType::hf);
             rerun = true;
+        } else if (one_of(st, DataType::ub, DataType::uw) && dt == DataType::u4) {
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: kealan-barbieri
body: fixed, and updated function name. 
created_at: 2025-03-07T19:43:46Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1544,162 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: atkassen
body: This doesn't seem accurate anymore.
created_at: 2025-03-10T16:57:59Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1544,162 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
+    // // Emulation sequence for  mov y:e3m0 x:hf
+    // and (nz)f0     null:uw x:uw    0xFFFF   /* preserve zeros */
+    // mov (nz)f0     t0:uw   x:uw
+    // (f0) add       t0:uw   t0:uw   -0x200   /* subtract 1/2 ULP */
+    // and (ne)f1     null:uw t0:uw   0x7FF    /* RTNE logic */
+    // shr            t0:uw   t0:uw   10       /* extract exponent */
+    // and            t0:uw   t0:uw   0x1F     /* remove sign bit */
+    // (f1) add       t0:uw   t0:uw   1        /* round up */
+    // and (ne)f1     null:uw x:uw    0x8000   /* check sign bit */
+    // (f0) add (sat) t0:uw   t0:uw   -12      /* adjust bias, clamp exponent low */
+    // sel (le)       t0:uw   t0:uw   7        /* clamp exponent high */
+    // (f1) or        t0:uw   t0:uw   8        /* insert sign bit */
+    // mov /* pack halfs into bytes */
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: atkassen
body: I don't think you need `f0` here. -0 and 2^-16 are handled differently than 0, but I assume still give correct results. We also apply RTNE unconditionally for [f32->bf16 emulation in reorder](https://github.com/oneapi-src/oneDNN/blob/29bba08321d5ba4dc8765ed0c5c0bff077d5d1e9/src/gpu/intel/jit/codegen/reorder.hpp#L327-L336), for example.
created_at: 2025-03-10T18:05:17Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1544,162 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
+    // // Emulation sequence for  mov y:e3m0 x:hf
+    // and (nz)f0     null:uw x:uw    0xFFFF   /* preserve zeros */
+    // mov (nz)f0     t0:uw   x:uw
+    // (f0) add       t0:uw   t0:uw   -0x200   /* subtract 1/2 ULP */
+    // and (ne)f1     null:uw t0:uw   0x7FF    /* RTNE logic */
+    // shr            t0:uw   t0:uw   10       /* extract exponent */
+    // and            t0:uw   t0:uw   0x1F     /* remove sign bit */
+    // (f1) add       t0:uw   t0:uw   1        /* round up */
+    // and (ne)f1     null:uw x:uw    0x8000   /* check sign bit */
+    // (f0) add (sat) t0:uw   t0:uw   -12      /* adjust bias, clamp exponent low */
+    // sel (le)       t0:uw   t0:uw   7        /* clamp exponent high */
+    // (f1) or        t0:uw   t0:uw   8        /* insert sign bit */
+    // mov /* pack halfs into bytes */
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: kealan-barbieri
body: applying ULP and bias adjustment to zero hf inputs gives incorrect results: 
```
run: --matmul --engine=gpu --memory-kind=buffer --dt=f4_e3m0:f4_e3m0:f4_e3m0 10x30:30x20
run ref: --matmul --engine=cpu --memory-kind=buffer 10x30:30x20
[   2][DST][0:2] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[   4][DST][0:4] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[   6][DST][0:6] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[   9][DST][0:9] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  20][DST][1:0] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  28][DST][1:8] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  32][DST][1:12] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  33][DST][1:13] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  41][DST][2:1] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  43][DST][2:3] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[COMPARE_STATS][DST]: trh=0 err_max_diff:      16 err_max_rdiff:      16 all_max_diff:      16 all_max_rdiff:      16
[PRIM_REF][INFO]: L2_size:2097152 bytes; per_core_L3_size:1966080 bytes; nthr:224; impl_name:brg_matmul:avx512_core
5:FAILED (errors:62 total:200) __REPRO: --matmul --engine=gpu --memory-kind=buffer --dt=f4_e3m0:f4_e3m0:f4_e3m0 10x30:30x20
create: --matmul --engine=gpu --memory-kind=buffer --dt=f4_e3m0:f4_e3m0:f4_e3m0 2048x1024:1024x512_n"DLRM:5*1"
oneDNN implementation: jit:gemm:any
[IMPL_FILTER] All implementations were skipped!
[IMPL_FILTER] All implementations were skipped!
CPU reference oneDNN implementation: brg_matmul:avx512_core
run: --matmul --engine=gpu --memory-kind=buffer --dt=f4_e3m0:f4_e3m0:f4_e3m0 2048x1024:1024x512_n"DLRM:5*1"
run ref: --matmul --engine=cpu --memory-kind=buffer 2048x1024:1024x512_n"DLRM:5*1"
[   5][DST][0:5] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[   6][DST][0:6] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  11][DST][0:11] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  12][DST][0:12] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  17][DST][0:17] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  24][DST][0:24] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  27][DST][0:27] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  31][DST][0:31] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  35][DST][0:35] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  38][DST][0:38] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[COMPARE_STATS][DST]: trh=0 err_max_diff:      16 err_max_rdiff:      16 all_max_diff:      16 all_max_rdiff:      16
[PRIM_REF][INFO]: L2_size:2097152 bytes; per_core_L3_size:1966080 bytes; nthr:224; impl_name:brg_matmul:avx512_core
6:FAILED (errors:319605 total:1048576) __REPRO: --matmul --engine=gpu --memory-kind=buffer --dt=f4_e3m0:f4_e3m0:f4_e3m0 2048x1024:1024x512_n"DLRM:5*1"
create: --matmul --engine=gpu --memory-kind=buffer --dt=f4_e3m0:f4_e3m0:f4_e3m0 2048x256:256x128_n"NCF:1*1"
oneDNN implementation: jit:gemm:any
[IMPL_FILTER] All implementations were skipped!
[IMPL_FILTER] All implementations were skipped!
CPU reference oneDNN implementation: brg_matmul:avx512_core
run: --matmul --engine=gpu --memory-kind=buffer --dt=f4_e3m0:f4_e3m0:f4_e3m0 2048x256:256x128_n"NCF:1*1"
run ref: --matmul --engine=cpu --memory-kind=buffer 2048x256:256x128_n"NCF:1*1"
[   5][DST][0:5] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[   8][DST][0:8] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[   9][DST][0:9] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  12][DST][0:12] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  14][DST][0:14] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  20][DST][0:20] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  22][DST][0:22] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  23][DST][0:23] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  24][DST][0:24] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[  25][DST][0:25] exp_f32:           0 exp:           0 got:          16 diff:      16 rdiff:      16
[COMPARE_STATS][DST]: trh=0 err_max_diff:      16 err_max_rdiff:      16 all_max_diff:      16 all_max_rdiff:      16
[PRIM_REF][INFO]: L2_size:2097152 bytes; per_core_L3_size:1966080 bytes; nthr:224; impl_name:brg_matmul:avx512_core
7:FAILED (errors:81130 total:262144) __REPRO: --matmul --engine=gpu --memory-kind=buffer --dt=f4_e3m0:f4_e3m0:f4_e3m0 2048x256:256x128_n"NCF:1*1"
create: --matmul --engine=gpu --memory-kind=buffer --dt=f4_e3m0:f4_e3m0:f4_e3m0 2048x128:128x64_n"NCF:2*1"
```
created_at: 2025-03-10T18:50:09Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1544,162 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: kealan-barbieri
body: Dropped, thanks.
created_at: 2025-03-10T18:52:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1544,162 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // Iterative emulation sequence for mov y:e3m0 x:hf 
+    // Control flow variables are shared and duplicates eliminated in later passes.
+    //
+    // // Emulation sequence for  mov y:e3m0 x:hf
+    // and (nz)f0     null:uw x:uw    0xFFFF   /* preserve zeros */
+    // mov (nz)f0     t0:uw   x:uw
+    // (f0) add       t0:uw   t0:uw   -0x200   /* subtract 1/2 ULP */
+    // and (ne)f1     null:uw t0:uw   0x7FF    /* RTNE logic */
+    // shr            t0:uw   t0:uw   10       /* extract exponent */
+    // and            t0:uw   t0:uw   0x1F     /* remove sign bit */
+    // (f1) add       t0:uw   t0:uw   1        /* round up */
+    // and (ne)f1     null:uw x:uw    0x8000   /* check sign bit */
+    // (f0) add (sat) t0:uw   t0:uw   -12      /* adjust bias, clamp exponent low */
+    // sel (le)       t0:uw   t0:uw   7        /* clamp exponent high */
+    // (f1) or        t0:uw   t0:uw   8        /* insert sign bit */
+    // mov /* pack halfs into bytes */
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: kealan-barbieri
body: discussed offline, refactored downscale algorithm to align.
created_at: 2025-03-11T18:29:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1544,165 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // // Emulation sequence for  mov y:e3m0 x:hf
+    // sel (lt)     t0:hf   x:hf    0x4c00:hf       /* clamp max val */
+    // mul          t0:hf   t0:hf   0x0c00:hf       /* round up */
+    // add          t0:uw   t0:uw   -0x200          /* subtract 1/2 ULP */
+    // and (nz)f0   null:uw t0:uw   0x7FF
+    // asr          t0:uw   t0:uw   10              /* extract exponent */
+    // (f0) add     t0:uw   t0:uw   1               /* round up */
+    // shl          t0:uw   t0:uw   12
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: atkassen
body: You can also simplify this down to
```
(f0) add t0:uw t0:uw 0x400
shl t0:uw t0:uw 2
```
created_at: 2025-03-11T22:09:14Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1479,6 +1544,165 @@ void CopyPlan::planEmulatedF4E2M1ToHF(CopyInstruction &i) {
         ie[7]->invalidate();
 }
 
+// Emulation sequence for hf->e3m0 conversion.
+void CopyPlan::planEmulatedHFToE3M0(CopyInstruction &i)
+{
+    // // Emulation sequence for  mov y:e3m0 x:hf
+    // sel (lt)     t0:hf   x:hf    0x4c00:hf       /* clamp max val */
+    // mul          t0:hf   t0:hf   0x0c00:hf       /* round up */
+    // add          t0:uw   t0:uw   -0x200          /* subtract 1/2 ULP */
+    // and (nz)f0   null:uw t0:uw   0x7FF
+    // asr          t0:uw   t0:uw   10              /* extract exponent */
+    // (f0) add     t0:uw   t0:uw   1               /* round up */
+    // shl          t0:uw   t0:uw   12
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: kealan-barbieri
body: done, thanks!
created_at: 2025-03-12T16:51:00Z
author_association: CONTRIBUTOR
diff_hunk: @@ -157,6 +156,7 @@ class CopyPlan
 protected:
     ngen::HW hw;
     bool systolicAvailable;
+    bool sharedAlloced = false;
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.hpp
user: login: atkassen
body: Is this used somewhere? Same with `allocSharedF4E3M0`.
created_at: 2025-03-12T17:22:55Z
author_association: CONTRIBUTOR
diff_hunk: @@ -157,6 +156,7 @@ class CopyPlan
 protected:
     ngen::HW hw;
     bool systolicAvailable;
+    bool sharedAlloced = false;
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.hpp
user: login: kealan-barbieri
body: no, both are leftovers from the iterative implementation, dropped. 
created_at: 2025-03-12T17:55:28Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1000,58 +1006,117 @@ void CopyPlan::planEmulatedHFToF4E2M1(CopyInstruction &i)
     ie[8]->ctrl = 0xCA;
 
     // Pack into byte.
-    ie[9]->op = Opcode::and_;
+
+    ie[9]->op = Opcode::shr;
     ie[9]->simd = simd;
     ie[9]->dst = tmp2;
     ie[9]->dst.type = DataType::uw;
     ie[9]->src0 = tmp2;
     ie[9]->src0.type = DataType::uw;
-    ie[9]->src1 = Immediate(0x00f0);
+    ie[9]->src1 = Immediate::uw(4);
 
-    ie[10]->op = Opcode::shr;
-    ie[10]->simd = simd/2;
-    ie[10]->dst = tmp2;
-    ie[10]->dst.type = DataType::uw;
-    ie[10]->dst.stride = 2;
+    ie[10]->op = Opcode::mov;
+    ie[10]->simd = simd;
+    ie[10]->dst = ddst;
+    ie[10]->dst.type = DataType::u4;
     ie[10]->src0 = tmp2;
     ie[10]->src0.type = DataType::uw;
-    ie[10]->src0.stride = 2;
-    ie[10]->src1 = Immediate::uw(4);
-
-    ie[11]->op = Opcode::or_;
-    ie[11]->simd = simd/2;
-    ie[11]->dst = tmp2;
-    ie[11]->dst.type = DataType::uw;
-    ie[11]->src0 = tmp2;
-    ie[11]->src0.offset = 1;
-    ie[11]->src0.type = DataType::uw;
-    ie[11]->src0.stride = 2;
-    ie[11]->src1 = tmp2;
-    ie[11]->src1.type = DataType::uw;
-    ie[11]->src1.stride = 2;
-
-    ie[12]->op = Opcode::mov;
-    ie[12]->simd = simd/2;
-    ie[12]->dst = tmp2;
-    ie[12]->dst.stride = 1;
-    ie[12]->dst.type = DataType::ub;
-    ie[12]->src0 = tmp2;
-    ie[12]->src0.stride = 2;
-    ie[12]->src0.type = DataType::ub;
-
-    ie[13]->op = Opcode::mov;
-    ie[13]->simd = simd/2;
-    ie[13]->dst = ddst;
-    if (ddst.inVS != 0)
-    ie[13]->dst.stride = ddst.inVS / ddst.inW;
-    ie[13]->dst.type = DataType::ub;
-    if (ie[13]->dst.offset != 0)
-            ie[13]->dst.offset /= 2;
-    ie[13]->src0 = tmp2;
-    ie[13]->src0.type = DataType::ub;
 
 }
 
+void CopyPlan::planBToI4(CopyInstruction &i)
+{
+    if (i.src0.neg || i.sat || i.hasCMod()) stub("Unsupported modifier");
+    int simd = i.simd;
+
+    auto ie = splitMultiple<6>(i);
+    auto tmp = newTemp(DataType::uw, simd, 1);
+
+    auto ddst = CopyOperand(i.dst);
+    auto invSrc = CopyOperand(i.src0);
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: atkassen
body: It doesn't look like `invSrc` is used anywhere.
created_at: 2025-03-17T16:05:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1000,58 +1006,117 @@ void CopyPlan::planEmulatedHFToF4E2M1(CopyInstruction &i)
     ie[8]->ctrl = 0xCA;
 
     // Pack into byte.
-    ie[9]->op = Opcode::and_;
+
+    ie[9]->op = Opcode::shr;
     ie[9]->simd = simd;
     ie[9]->dst = tmp2;
     ie[9]->dst.type = DataType::uw;
     ie[9]->src0 = tmp2;
     ie[9]->src0.type = DataType::uw;
-    ie[9]->src1 = Immediate(0x00f0);
+    ie[9]->src1 = Immediate::uw(4);
 
-    ie[10]->op = Opcode::shr;
-    ie[10]->simd = simd/2;
-    ie[10]->dst = tmp2;
-    ie[10]->dst.type = DataType::uw;
-    ie[10]->dst.stride = 2;
+    ie[10]->op = Opcode::mov;
+    ie[10]->simd = simd;
+    ie[10]->dst = ddst;
+    ie[10]->dst.type = DataType::u4;
     ie[10]->src0 = tmp2;
     ie[10]->src0.type = DataType::uw;
-    ie[10]->src0.stride = 2;
-    ie[10]->src1 = Immediate::uw(4);
-
-    ie[11]->op = Opcode::or_;
-    ie[11]->simd = simd/2;
-    ie[11]->dst = tmp2;
-    ie[11]->dst.type = DataType::uw;
-    ie[11]->src0 = tmp2;
-    ie[11]->src0.offset = 1;
-    ie[11]->src0.type = DataType::uw;
-    ie[11]->src0.stride = 2;
-    ie[11]->src1 = tmp2;
-    ie[11]->src1.type = DataType::uw;
-    ie[11]->src1.stride = 2;
-
-    ie[12]->op = Opcode::mov;
-    ie[12]->simd = simd/2;
-    ie[12]->dst = tmp2;
-    ie[12]->dst.stride = 1;
-    ie[12]->dst.type = DataType::ub;
-    ie[12]->src0 = tmp2;
-    ie[12]->src0.stride = 2;
-    ie[12]->src0.type = DataType::ub;
-
-    ie[13]->op = Opcode::mov;
-    ie[13]->simd = simd/2;
-    ie[13]->dst = ddst;
-    if (ddst.inVS != 0)
-    ie[13]->dst.stride = ddst.inVS / ddst.inW;
-    ie[13]->dst.type = DataType::ub;
-    if (ie[13]->dst.offset != 0)
-            ie[13]->dst.offset /= 2;
-    ie[13]->src0 = tmp2;
-    ie[13]->src0.type = DataType::ub;
 
 }
 
+void CopyPlan::planBToI4(CopyInstruction &i)
+{
+    if (i.src0.neg || i.sat || i.hasCMod()) stub("Unsupported modifier");
+    int simd = i.simd;
+
+    auto ie = splitMultiple<6>(i);
+    auto tmp = newTemp(DataType::uw, simd, 1);
+
+    auto ddst = CopyOperand(i.dst);
+    auto invSrc = CopyOperand(i.src0);
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: kealan-barbieri
body: dropped, thanks.
created_at: 2025-03-17T16:22:47Z
author_association: CONTRIBUTOR
diff_hunk: @@ -994,55 +1000,113 @@ void CopyPlan::planEmulatedHFToF4E2M1(CopyInstruction &i)
     ie[8]->src1 = Immediate::uw(8);
 
     // Pack into byte.
-    ie[9]->op = Opcode::and_;
+
+    ie[9]->op = Opcode::shr;
     ie[9]->simd = simd;
     ie[9]->dst = tmp;
     ie[9]->dst.type = DataType::uw;
     ie[9]->src0 = tmp;
     ie[9]->src0.type = DataType::uw;
-    ie[9]->src1 = Immediate(0x00f0);
+    ie[9]->src1 = Immediate::uw(4);
path: src/gpu/intel/jit/gemm/generator/pieces/copy_plan.cpp
user: login: atkassen
body: ```suggestion
    ie[8]->src1 = Immediate::uw(12);
```
(and adjust indices below/split size above)
created_at: 2025-03-17T17:03:21Z
author_association: CONTRIBUTOR

PR comments:
user: login: kealan-barbieri
created_at: 2025-02-27T01:43:37Z
body: make test
enable test_device_cpu
enable test_device_gpu
disable benchdnn_all
enable benchdnn matmul
enable benchdnn ip
author_association: CONTRIBUTOR
user: login: kealan-barbieri
created_at: 2025-02-27T17:26:50Z
body: make test
disable test_device_cpu
enable test_device_gpu
disable benchdnn_all
enable benchdnn_matmul
enable benchdnn_ip

author_association: CONTRIBUTOR
user: login: kealan-barbieri
created_at: 2025-03-05T01:56:41Z
body: make test
disable test_device_cpu
enable test_device_gpu
disable benchdnn_all
enable benchdnn_matmul
enable benchdnn_ip
author_association: CONTRIBUTOR
user: login: kealan-barbieri
created_at: 2025-03-07T19:43:29Z
body: make test
disable test_device_cpu
enable test_device_gpu
disable benchdnn_all
enable benchdnn_matmul
enable benchdnn_ip
author_association: CONTRIBUTOR
user: login: kealan-barbieri
created_at: 2025-03-10T17:08:05Z
body: make test
disable test_device_cpu
enable test_device_gpu
disable benchdnn_all
enable benchdnn_matmul
enable benchdnn_ip
author_association: CONTRIBUTOR
user: login: kealan-barbieri
created_at: 2025-03-11T18:29:11Z
body: make test
disable test_device_cpu
enable test_device_gpu
disable benchdnn_all
enable benchdnn_matmul
enable benchdnn_ip
author_association: CONTRIBUTOR
user: login: kealan-barbieri
created_at: 2025-03-12T22:58:26Z
body: make test
disable test_device_cpu
enable test_device_gpu
disable benchdnn_all
enable benchdnn_matmul
enable benchdnn_ip
author_association: CONTRIBUTOR
user: login: kealan-barbieri
created_at: 2025-03-17T16:58:02Z
body: make test
disable test_device_cpu
enable test_device_gpu
disable benchdnn_all
enable benchdnn_matmul
enable benchdnn_ip
author_association: CONTRIBUTOR