Repository: onnx/onnx-mlir
Pull request #3117: Support QLinearMatMul on CPU
Author: login: tungld
Created: 2025-04-11T05:36:52Z | Merged: 2025-04-16T03:23:55Z

This PR is to support QLinearMatMul on CPU. In particular it lowers ONNX.QLinearMatMul to Krnl dialect by rewriting the operation into ONNX.MatMul of i32 element type.

Currently only i8, ui8 and f32 data types are supported, and the backend tests for these data types are enabled.


Diff:
diff --git a/docs/SupportedONNXOps-cpu.md b/docs/SupportedONNXOps-cpu.md
index 572e14a7ab0..418dc5cfd58 100644
--- a/docs/SupportedONNXOps-cpu.md
+++ b/docs/SupportedONNXOps-cpu.md
@@ -145,7 +145,7 @@ Onnx-mlir currently supports ONNX operations targeting up to opset 22. Limitatio
 | **Pad** |6 - * |axes input not supported. Does not support int4 and uint4. | |
 | **Pow** |7 - * |No support for power with integer types. | |
 | **QLinearConv** |none | | | |
-| **QLinearMatMul** |none | | | |
+| **QLinearMatMul** |10 - * |Only support i8, ui8 and f32. | |
 | **QuantizeLinear** |10 - * |Does not support per-axis and i8 quantization. Does not support int4 and uint4. | |
 | **RNN** |7 - * |W, B and R must be constants. | |
 | **RandomNormal** |none | | | |
diff --git a/src/Accelerators/NNPA/Conversion/ONNXToZHigh/ONNXToZHigh.cpp b/src/Accelerators/NNPA/Conversion/ONNXToZHigh/ONNXToZHigh.cpp
index 921fba751d2..e29150cc347 100644
--- a/src/Accelerators/NNPA/Conversion/ONNXToZHigh/ONNXToZHigh.cpp
+++ b/src/Accelerators/NNPA/Conversion/ONNXToZHigh/ONNXToZHigh.cpp
@@ -386,25 +386,6 @@ static void preComputeBias(MultiDialectBuilder<OnnxBuilder> &create, Value RSa,
   Zqctilde = qcOp.getResult(2);
 }
 
-static Value getOrCastToI8(Value val, MultiDialectBuilder<OnnxBuilder> &create,
-    bool simpleCast = false) {
-  if (!getElementType(val.getType()).isUnsignedInteger())
-    return val;
-
-  Type i8Ty = create.getBuilder().getI8Type();
-  if (simpleCast)
-    return create.onnx.cast(val, i8Ty);
-
-  // Use int16 to avoid integer overflow.
-  Type i16Ty = create.getBuilder().getI16Type();
-  auto cst128Attr = DenseElementsAttr::get(
-      RankedTensorType::get({}, i16Ty), static_cast<int16_t>(128));
-  Value valI16 = create.onnx.cast(val, i16Ty);
-  valI16 = create.onnx.sub(valI16, create.onnx.constant(cst128Attr));
-  Value valI8 = create.onnx.cast(valI16, i8Ty);
-  return valI8;
-}
-
 // Dynamic quantization helper to match and rewrite values A, B, C of A*B+C.
 class DynQuantI8PatternHelper {
 public:
@@ -798,8 +779,8 @@ class replaceONNXMatMulIntegerPattern
     // Prepare inputs for zhigh QuantizedMatMul.
 
     // I8 tensors
-    Value AI8 = getOrCastToI8(mmiOp.getA(), create, true);
-    Value BI8 = getOrCastToI8(mmiOp.getB(), create, true);
+    Value AI8 = create.onnx.getOrCastToI8(mmiOp.getA(), true);
+    Value BI8 = create.onnx.getOrCastToI8(mmiOp.getB(), true);
 
     // Zero points in f32.
     Value AZeroPointI8 = mmiOp.getAZeroPoint();
@@ -809,7 +790,7 @@ class replaceONNXMatMulIntegerPattern
           RankedTensorType::get({}, getElementType(AZeroPointI8.getType())),
           AZeroPointI8, {zeroI64});
     }
-    AZeroPointI8 = getOrCastToI8(AZeroPointI8, create, true);
+    AZeroPointI8 = create.onnx.getOrCastToI8(AZeroPointI8, true);
     Value AZeroPointF32 = create.onnx.cast(AZeroPointI8, f32Ty);
     // TESTING: minus zeropoint in advance to cancel out the software part of
     // zdnn quantized matmul.
@@ -822,7 +803,7 @@ class replaceONNXMatMulIntegerPattern
           RankedTensorType::get({}, getElementType(BZeroPointI8.getType())),
           BZeroPointI8, {zeroI64});
     }
-    BZeroPointI8 = getOrCastToI8(BZeroPointI8, create, true);
+    BZeroPointI8 = create.onnx.getOrCastToI8(BZeroPointI8, true);
     Value BZeroPointF32 = create.onnx.cast(BZeroPointI8, f32Ty);
     // TESTING: minus zeropoint in advance to cancel out the software part of
     // zdnn quantized matmul.
@@ -964,10 +945,10 @@ class replaceMatMulIntegerSubGraphFromMulPattern
             RankedTensorType::get({}, getElementType(AZeroPointI8.getType())),
             AZeroPointI8, {zeroI64});
       }
-      AZeroPointI8 = getOrCastToI8(AZeroPointI8, create, true);
+      AZeroPointI8 = create.onnx.getOrCastToI8(AZeroPointI8, true);
       Value AZeroPointF32 = create.onnx.cast(AZeroPointI8, f32Ty);
       Value ARecScale = create.onnx.reciprocal(AScale);
-      AI8 = getOrCastToI8(AI8, create, true);
+      AI8 = create.onnx.getOrCastToI8(AI8, true);
       // Stickify the quantized input A to ztensor format.
       qAOp = rewriter.create<ZHighQuantizedStickOp>(loc, AI8, ARecScale,
           AZeroPointF32, aLayoutAttr, rewriter.getStringAttr(QTYPE_INT8));
@@ -1418,19 +1399,19 @@ class replaceONNXQLinearMatMulPattern
     }
 
     // zdnn supports signed int8, convert unsigned int8 inputs to signed int8.
-    Value AI8 = getOrCastToI8(A, create);
-    Value BI8 = getOrCastToI8(B, create);
+    Value AI8 = create.onnx.getOrCastToI8(A);
+    Value BI8 = create.onnx.getOrCastToI8(B);
 
     Value ARecScale = create.onnx.reciprocal(AScale);
-    Value AZeroPointI8 = getOrCastToI8(AZeroPoint, create);
+    Value AZeroPointI8 = create.onnx.getOrCastToI8(AZeroPoint);
     Value AZeroPointF32 = create.onnx.cast(AZeroPointI8, f32Ty);
 
     Value BRecScale = create.onnx.reciprocal(BScale);
-    Value BZeroPointI8 = getOrCastToI8(BZeroPoint, create);
+    Value BZeroPointI8 = create.onnx.getOrCastToI8(BZeroPoint);
     Value BZeroPointF32 = create.onnx.cast(BZeroPointI8, f32Ty);
 
     Value YRecScale = create.onnx.reciprocal(YScale);
-    Value YZeroPointI8 = getOrCastToI8(YZeroPoint, create);
+    Value YZeroPointI8 = create.onnx.getOrCastToI8(YZeroPoint);
     Value YZeroPointF32 = create.onnx.cast(YZeroPointI8, f32Ty);
 
     // Stickify AI8, Transform AI8 into zTensor format.
diff --git a/src/Conversion/ONNXToKrnl/CMakeLists.txt b/src/Conversion/ONNXToKrnl/CMakeLists.txt
index 6a68f3cf2ac..51b5cfe66d7 100644
--- a/src/Conversion/ONNXToKrnl/CMakeLists.txt
+++ b/src/Conversion/ONNXToKrnl/CMakeLists.txt
@@ -22,6 +22,7 @@ add_onnx_mlir_library(OMONNXToKrnl
   Math/LRN.cpp
   Math/MatMul.cpp
   Math/MatMulInteger.cpp
+  Math/QLinearMatMul.cpp
   Math/RandomNormal.cpp
   Math/RandomNormalLike.cpp
   Math/Reduction.cpp
diff --git a/src/Conversion/ONNXToKrnl/ConvertONNXToKrnl.cpp b/src/Conversion/ONNXToKrnl/ConvertONNXToKrnl.cpp
index f901d679317..8a3640cb87e 100644
--- a/src/Conversion/ONNXToKrnl/ConvertONNXToKrnl.cpp
+++ b/src/Conversion/ONNXToKrnl/ConvertONNXToKrnl.cpp
@@ -220,6 +220,7 @@ void populateONNXToKrnlConversionPattern(RewritePatternSet &patterns,
   populateLoweringONNXRandomNormalOpPattern(patterns, typeConverter, ctx);
   populateLoweringONNXRandomNormalLikeOpPattern(patterns, typeConverter, ctx);
   populateLoweringONNXLRNOpPattern(patterns, typeConverter, ctx);
+  populateLoweringONNXQLinearMatMulOpPattern(patterns, typeConverter, ctx);
   // ML
   populateLoweringONNXCategoryMapperOpPattern(patterns, typeConverter, ctx);
   // ObjectDetection
diff --git a/src/Conversion/ONNXToKrnl/Math/QLinearMatMul.cpp b/src/Conversion/ONNXToKrnl/Math/QLinearMatMul.cpp
new file mode 100644
index 00000000000..846ece03f3d
--- /dev/null
+++ b/src/Conversion/ONNXToKrnl/Math/QLinearMatMul.cpp
@@ -0,0 +1,150 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+//===--------- QLinearMatMul.cpp - Lowering QLinearMatMul Op --------------===//
+//
+// Copyright 2019-2025 The IBM Research Authors.
+//
+// =============================================================================
+//
+// This file lowers the ONNX QLinearMatMul Operator to Krnl dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/Conversion/ONNXToKrnl/ONNXToKrnlCommon.hpp"
+#include "src/Dialect/Krnl/DialectBuilder.hpp"
+#include "src/Dialect/Krnl/KrnlHelper.hpp"
+#include "src/Dialect/Mlir/DialectBuilder.hpp"
+#include "src/Dialect/Mlir/IndexExpr.hpp"
+#include "src/Dialect/ONNX/ONNXOps/ShapeHelper.hpp"
+
+using namespace mlir;
+
+namespace onnx_mlir {
+
+struct ONNXQLinearMatMulOpLowering
+    : public OpConversionPattern<ONNXQLinearMatMulOp> {
+public:
+  ONNXQLinearMatMulOpLowering(TypeConverter &typeConverter, MLIRContext *ctx)
+      : OpConversionPattern(typeConverter, ctx) {}
+
+  LogicalResult matchAndRewrite(ONNXQLinearMatMulOp qlmmOp,
+      ONNXQLinearMatMulOpAdaptor adaptor,
+      ConversionPatternRewriter &rewriter) const final {
+    using LocalDialectBuilder =
+        MultiDialectBuilder<IndexExprBuilderForKrnl, OnnxBuilder>;
+    Operation *op = qlmmOp.getOperation();
+    Location loc = ONNXLoc<ONNXQLinearMatMulOp>(op);
+    LocalDialectBuilder create(rewriter, loc);
+
+    ValueRange operands = adaptor.getOperands();
+    Value A = adaptor.getA();
+    Value aScale = adaptor.getAScale();
+    Value aZeroPoint = adaptor.getAZeroPoint();
+    Value B = adaptor.getB();
+    Value bScale = adaptor.getBScale();
+    Value bZeroPoint = adaptor.getBZeroPoint();
+    Value yScale = adaptor.getYScale();
+    Value yZeroPoint = adaptor.getYZeroPoint();
+
+    // Now only support integer8 for inputs and zeropoints, and support float32
+    // for scale.
+    if (!getElementType(A.getType()).isInteger(8))
+      return failure();
+    if (!getElementType(B.getType()).isInteger(8))
+      return failure();
+    if (!getElementType(aScale.getType()).isF32())
+      return failure();
+    if (!getElementType(bScale.getType()).isF32())
+      return failure();
+    if (!getElementType(yScale.getType()).isF32())
+      return failure();
+    if (!getElementType(aZeroPoint.getType()).isInteger(8))
+      return failure();
+    if (!getElementType(bZeroPoint.getType()).isInteger(8))
+      return failure();
+    if (!getElementType(yZeroPoint.getType()).isInteger(8))
+      return failure();
+
+    // Common types.
+    Type i8Ty = rewriter.getI8Type();
+    Type i32Ty = rewriter.getI32Type();
+    Type f32Ty = rewriter.getF32Type();
+    auto resMemRefType = dyn_cast<MemRefType>(
+        typeConverter->convertType(qlmmOp.getResult().getType()));
+    Type resElementType = resMemRefType.getElementType();
+
+    // Get shape.
+    ONNXQLinearMatMulOpShapeHelper shapeHelper(op, operands, &create.krnlIE);
+    shapeHelper.computeShapeAndAssertOnFailure();
+
+    Value cst128;
+    if (resElementType.isUnsignedInteger(8)) {
+      auto cst128Attr = DenseElementsAttr::get(
+          RankedTensorType::get({}, i32Ty), static_cast<int32_t>(128));
+      cst128 = create.onnx.constant(cst128Attr);
+    }
+
+    // Prepare input A.
+    Value AI8 = create.onnx.getOrCastToI8(A);
+    Value AI32 = create.onnx.cast(AI8, i32Ty);
+    auto aZeroPointType = mlir::cast<ShapedType>(aZeroPoint.getType());
+    int64_t aZeroPointRank = aZeroPointType.getRank();
+    Value aZeroPointI8 = create.onnx.getOrCastToI8(aZeroPoint);
+    Value aZeroPointI32 = create.onnx.cast(aZeroPointI8, i32Ty);
+    // If broadcasting, e.g. A is [MxK], zeroPoint is [M], M != 1.
+    // Unsqueeze zeroPoint to [Mx1] to make shapes compatible.
+    // There is no need to handle scalar zeroPoint (e.g. tensor<dtype> or
+    // tensor<1xdtype>), which is always true for broadcasting.
+    if ((aZeroPointRank == 1) && (aZeroPointType.getShape()[0] != 1)) {
+      SmallVector<int64_t, 4> unsqueezeShape(aZeroPointType.getShape());
+      unsqueezeShape.emplace_back(1);
+      aZeroPointI32 =
+          create.onnx.unsqueeze(RankedTensorType::get(unsqueezeShape, i32Ty),
+              aZeroPointI32, create.onnx.constantInt64({aZeroPointRank}));
+    }
+    AI32 = create.onnx.sub(AI32, aZeroPointI32);
+
+    // Prepare input B.
+    Value BI8 = create.onnx.getOrCastToI8(B);
+    Value BI32 = create.onnx.cast(BI8, i32Ty);
+    // K is the broadcating dim: [KxN] - [N] = [KxN] - [1xN]
+    Value bZeroPointI8 = create.onnx.getOrCastToI8(bZeroPoint);
+    Value bZeroPointI32 = create.onnx.cast(bZeroPointI8, i32Ty);
+    BI32 = create.onnx.sub(BI32, bZeroPointI32);
+
+    // Prepare output Y
+    Value yZeroPointI8 = create.onnx.getOrCastToI8(yZeroPoint);
+    Value yZeroPointI32 = create.onnx.cast(yZeroPointI8, i32Ty);
+
+    // Emit MatMul.
+    Value resI32 = create.onnx.matmul(
+        RankedTensorType::get(resMemRefType.getShape(), i32Ty), AI32, BI32);
+
+    // Scale the output.
+    Value resF32 = create.onnx.cast(resI32, f32Ty);
+    Value scale = create.onnx.div(create.onnx.mul(aScale, bScale), yScale);
+    resF32 = create.onnx.mul(resF32, scale);
+
+    // Saturate and add zero point.
+    Value roundToEven = create.onnx.round(resF32);
+    resI32 = create.onnx.cast(roundToEven, i32Ty);
+    resI32 = create.onnx.add(resI32, yZeroPointI32);
+    if (resElementType.isUnsignedInteger(8)) {
+      resI32 = create.onnx.add(resI32, cst128);
+      resI32 = create.onnx.cast(resI32, i8Ty);
+    }
+    Value res = create.onnx.cast(resI32, resElementType);
+
+    rewriter.replaceOp(op, {create.onnx.toMemref(res)});
+    return success();
+  }
+};
+
+void populateLoweringONNXQLinearMatMulOpPattern(RewritePatternSet &patterns,
+    TypeConverter &typeConverter, MLIRContext *ctx) {
+  patterns.insert<ONNXQLinearMatMulOpLowering>(typeConverter, ctx);
+}
+
+} // namespace onnx_mlir
diff --git a/src/Conversion/ONNXToKrnl/ONNXToKrnlCommon.hpp b/src/Conversion/ONNXToKrnl/ONNXToKrnlCommon.hpp
index c4dea75fb6d..4e2164b846d 100644
--- a/src/Conversion/ONNXToKrnl/ONNXToKrnlCommon.hpp
+++ b/src/Conversion/ONNXToKrnl/ONNXToKrnlCommon.hpp
@@ -342,6 +342,8 @@ void populateLoweringONNXMatMulOpPattern(mlir::RewritePatternSet &,
     bool enableTiling, bool enableSIMD, bool enableParallel);
 void populateLoweringONNXMatMulIntegerOpPattern(
     mlir::RewritePatternSet &, mlir::TypeConverter &, mlir::MLIRContext *);
+void populateLoweringONNXQLinearMatMulOpPattern(
+    mlir::RewritePatternSet &, mlir::TypeConverter &, mlir::MLIRContext *);
 void populateLoweringONNXRandomNormalOpPattern(
     mlir::RewritePatternSet &, mlir::TypeConverter &, mlir::MLIRContext *);
 void populateLoweringONNXRandomNormalLikeOpPattern(
diff --git a/src/Dialect/ONNX/DialectBuilder.cpp b/src/Dialect/ONNX/DialectBuilder.cpp
index f86b4b84ad2..5b5bce57dd6 100644
--- a/src/Dialect/ONNX/DialectBuilder.cpp
+++ b/src/Dialect/ONNX/DialectBuilder.cpp
@@ -808,6 +808,28 @@ Value OnnxBuilder::foldOrEmitONNXTransposeOp(
   }
 }
 
+//===----------------------------------------------------------------------===//
+// Helper for quantization
+//===----------------------------------------------------------------------===//
+Value OnnxBuilder::getOrCastToI8(Value val, bool simpleCast) {
+  if (!getElementType(val.getType()).isUnsignedInteger())
+    return val;
+
+  Type i8Ty = b().getI8Type();
+  if (simpleCast)
+    return cast(val, i8Ty);
+
+  // Use int16 to avoid integer overflow.
+  Type i16Ty = b().getI16Type();
+  auto cst128Attr = DenseElementsAttr::get(
+      RankedTensorType::get({}, i16Ty), static_cast<int16_t>(128));
+  Value cst128 = constant(cst128Attr);
+  Value valI16 = cast(val, i16Ty);
+  valI16 = sub(valI16, cst128);
+  Value valI8 = cast(valI16, i8Ty);
+  return valI8;
+}
+
 // =============================================================================
 // IndexExpr Builder for Analysis
 // =============================================================================
diff --git a/src/Dialect/ONNX/DialectBuilder.hpp b/src/Dialect/ONNX/DialectBuilder.hpp
index 5fd232138fb..5e4efc8209e 100644
--- a/src/Dialect/ONNX/DialectBuilder.hpp
+++ b/src/Dialect/ONNX/DialectBuilder.hpp
@@ -308,6 +308,13 @@ struct OnnxBuilder : DialectBuilder {
       mlir::Type resultType, mlir::Value input, mlir::ArrayAttr permAttr,
       DenseElementsAttrGetter getDenseElementAttrFromConstValue);
 
+  // ===========================================================================
+  // Quantization support.
+  // ===========================================================================
+
+  /// Get or cast a value to i8 tensor.
+  mlir::Value getOrCastToI8(mlir::Value val, bool simpleCast = false);
+
 private:
   mlir::IntegerAttr getSignedInt64Attr(int64_t n) const;
 };
diff --git a/src/Dialect/ONNX/ONNXOps/OpHelper.cpp b/src/Dialect/ONNX/ONNXOps/OpHelper.cpp
index 7f260f2e99f..2260d778d39 100644
--- a/src/Dialect/ONNX/ONNXOps/OpHelper.cpp
+++ b/src/Dialect/ONNX/ONNXOps/OpHelper.cpp
@@ -890,5 +890,4 @@ bool isElementAttrUninitializedDenseResource(ElementsAttr elementsAttr) {
   return denseResourceElementsAttr &&
          !denseResourceElementsAttr.getRawHandle().getBlob();
 }
-
 } // namespace onnx_mlir
diff --git a/test/backend/inference_backend.py b/test/backend/inference_backend.py
index f3059af9311..6f5de693d4c 100644
--- a/test/backend/inference_backend.py
+++ b/test/backend/inference_backend.py
@@ -2328,6 +2328,29 @@ def get_test_models():
             DYNAMIC_SHAPE: {-1: {-1}},
             CONSTANT_INPUT: {-1},
         },
+        # ==OP== QLinearMatMul
+        # ==MIN== 10
+        # ==LIM== Only support i8, ui8 and f32.
+        "test_qlinearmatmul_2D_int8_float32_cpu": {
+            STATIC_SHAPE: {},
+            DYNAMIC_SHAPE: {-1: {-1}},
+            CONSTANT_INPUT: {-1},
+        },
+        "test_qlinearmatmul_2D_uint8_float32_cpu": {
+            STATIC_SHAPE: {},
+            DYNAMIC_SHAPE: {-1: {-1}},
+            CONSTANT_INPUT: {-1},
+        },
+        "test_qlinearmatmul_3D_int8_float32_cpu": {
+            STATIC_SHAPE: {},
+            DYNAMIC_SHAPE: {-1: {-1}},
+            CONSTANT_INPUT: {-1},
+        },
+        "test_qlinearmatmul_3D_uint8_float32_cpu": {
+            STATIC_SHAPE: {},
+            DYNAMIC_SHAPE: {-1: {-1}},
+            CONSTANT_INPUT: {-1},
+        },
         # ==OP== Range
         # ==MIN== 11
         "test_range_float_type_positive_delta_cpu": {
diff --git a/test/mlir/accelerators/nnpa/conversion/onnx-to-zhigh/qlinearmatmul.mlir b/test/mlir/accelerators/nnpa/conversion/onnx-to-zhigh/qlinearmatmul.mlir
index 2a7a11637bb..edbb01289b2 100644
--- a/test/mlir/accelerators/nnpa/conversion/onnx-to-zhigh/qlinearmatmul.mlir
+++ b/test/mlir/accelerators/nnpa/conversion/onnx-to-zhigh/qlinearmatmul.mlir
@@ -55,7 +55,7 @@ func.func @qlinearmatmul_ui8_f32(%arg0: tensor<2x4xui8> {onnx.name = "a"}, %arg1
 // CHECK:           [[VAR_22_:%.+]] = "onnx.Cast"([[VAR_21_]]) {saturate = 1 : si64, to = f32} : (tensor<i8>) -> tensor<f32>
 // CHECK:           [[VAR_Out_:%.+]], [[VAR_RecScale_:%.+]], [[VAR_Offset_:%.+]] = "zhigh.QuantizedStick"([[VAR_4_]], [[VAR_8_]], [[VAR_12_]]) {layout = "2D", quantized_type = "INT8", sym_mode = 0 : i64} : (tensor<2x4xi8>, tensor<f32>, tensor<f32>) -> (tensor<2x4xi8, #zhigh.layout<{dataLayout = "2D", quantizedType = "INT8"}>>, tensor<f32>, tensor<f32>)
 // CHECK:           [[VAR_Out_0_:%.+]], [[VAR_RecScale_1_:%.+]], [[VAR_Offset_2_:%.+]] = "zhigh.QuantizedStick"([[VAR_7_]], [[VAR_13_]], [[VAR_17_]]) {layout = "2D", quantized_type = "WEIGHTS", sym_mode = 0 : i64} : (tensor<4x3xi8>, tensor<f32>, tensor<f32>) -> (tensor<4x3xi8, #zhigh.layout<{dataLayout = "2D", quantizedType = "WEIGHTS"}>>, tensor<f32>, tensor<f32>)
-// CHECK:           [[VAR_Out_3_:%.+]], [[VAR_OutRecScale_:%.+]], [[VAR_OutOffset_:%.+]] = "zhigh.QuantizedMatMul"([[VAR_Out_]], [[VAR_RecScale_]], [[VAR_Offset_]], [[VAR_Out_]]_0, [[VAR_RecScale_]]_1, [[VAR_Offset_]]_2, [[VAR_1_]], [[VAR_1_]], [[VAR_1_]], [[VAR_1_]]8, [[VAR_22_]]) {DequantizeOutput = 0 : si64, DisableClipping = -1 : si64, PreComputedBias = 0 : si64} : (tensor<2x4xi8, #zhigh.layout<{dataLayout = "2D", quantizedType = "INT8"}>>, tensor<f32>, tensor<f32>, tensor<4x3xi8, #zhigh.layout<{dataLayout = "2D", quantizedType = "WEIGHTS"}>>, tensor<f32>, tensor<f32>, none, none, none, tensor<f32>, tensor<f32>) -> (tensor<2x3xf16, #zhigh.layout<{dataLayout = "2D", quantizedType = "DLFLOAT16"}>>, tensor<f32>, tensor<f32>)
+// CHECK:           [[VAR_Out_3_:%.+]], [[VAR_OutRecScale_:%.+]], [[VAR_OutOffset_:%.+]] = "zhigh.QuantizedMatMul"([[VAR_Out_]], [[VAR_RecScale_]], [[VAR_Offset_]], [[VAR_Out_0_]], [[VAR_RecScale_1_]], [[VAR_Offset_2_]], [[VAR_1_]], [[VAR_1_]], [[VAR_1_]], [[VAR_18_]], [[VAR_22_]]) {DequantizeOutput = 0 : si64, DisableClipping = -1 : si64, PreComputedBias = 0 : si64} : (tensor<2x4xi8, #zhigh.layout<{dataLayout = "2D", quantizedType = "INT8"}>>, tensor<f32>, tensor<f32>, tensor<4x3xi8, #zhigh.layout<{dataLayout = "2D", quantizedType = "WEIGHTS"}>>, tensor<f32>, tensor<f32>, none, none, none, tensor<f32>, tensor<f32>) -> (tensor<2x3xf16, #zhigh.layout<{dataLayout = "2D", quantizedType = "DLFLOAT16"}>>, tensor<f32>, tensor<f32>)
 // CHECK:           [[VAR_23_:%.+]] = "zhigh.Unstick"([[VAR_Out_3_]]) : (tensor<2x3xf16, #zhigh.layout<{dataLayout = "2D", quantizedType = "DLFLOAT16"}>>) -> tensor<2x3xf32>
 // CHECK:           [[VAR_24_:%.+]] = "onnx.Cast"([[VAR_23_]]) {saturate = 1 : si64, to = i16} : (tensor<2x3xf32>) -> tensor<2x3xi16>
 // CHECK:           [[VAR_25_:%.+]] = "onnx.Add"([[VAR_24_]], [[VAR_0_]]) : (tensor<2x3xi16>, tensor<i16>) -> tensor<2x3xi16>
diff --git a/test/mlir/conversion/onnx_to_krnl/Math/QLinearMatMul_with_canonicalize.mlir b/test/mlir/conversion/onnx_to_krnl/Math/QLinearMatMul_with_canonicalize.mlir
new file mode 100644
index 00000000000..bc3eecc9711
--- /dev/null
+++ b/test/mlir/conversion/onnx_to_krnl/Math/QLinearMatMul_with_canonicalize.mlir
@@ -0,0 +1,325 @@
+// RUN: onnx-mlir-opt --shape-inference --convert-onnx-to-krnl --canonicalize %s -split-input-file | FileCheck %s
+
+func.func @qlinearmatmul_i8_f32(%arg0: tensor<16x32xi8>, %arg1: tensor<1xf32>, %arg2: tensor<1xi8>, %arg3: tensor<32x64xi8>, %arg4: tensor<1xf32>, %arg5: tensor<1xi8>, %arg6: tensor<1xf32>, %arg7: tensor<1xi8>) -> (tensor<16x64xi8>) {
+    %0 = "onnx.QLinearMatMul"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %arg6, %arg7) : (tensor<16x32xi8>, tensor<1xf32>, tensor<1xi8>, tensor<32x64xi8>, tensor<1xf32>, tensor<1xi8>, tensor<1xf32>, tensor<1xi8>) -> tensor<16x64xi8>
+    return %0 : tensor<16x64xi8>
+
+// CHECK-LABEL:  func.func @qlinearmatmul_i8_f32
+// CHECK-SAME:   ([[PARAM_0_:%.+]]: memref<16x32xi8>, [[PARAM_1_:%.+]]: memref<1xf32>, [[PARAM_2_:%.+]]: memref<1xi8>, [[PARAM_3_:%.+]]: memref<32x64xi8>, [[PARAM_4_:%.+]]: memref<1xf32>, [[PARAM_5_:%.+]]: memref<1xi8>, [[PARAM_6_:%.+]]: memref<1xf32>, [[PARAM_7_:%.+]]: memref<1xi8>) -> memref<16x64xi8> {
+// CHECK-DAG:       [[CST_5_dot_000000_:%.+]] = arith.constant 5.000000e-01 : f32
+// CHECK-DAG:       [[CST_2_dot_000000_:%.+]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG:       [[CST_1_dot_000000_:%.+]] = arith.constant 1.000000e+00 : f32
+// CHECK-DAG:       [[CST_0_:%.+]] = arith.constant 0 : i32
+// CHECK-DAG:       [[CST_0_1_:%.+]] = arith.constant 0 : index
+// CHECK-DAG:       [[RES_:%.+]] = memref.alloc() {{.*}}: memref<16x32xi32>
+// CHECK-DAG:       [[LOOP_0_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_0_]]#0, [[LOOP_0_]]#1) with ([[LOOP_0_]]#0 -> [[I_0_:%.+]] = 0 to 16, [[LOOP_0_]]#1 -> [[I_1_:%.+]] = 0 to 32){
+// CHECK:             [[VAR_20_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_0_]]#0, [[LOOP_0_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_:%.+]] = krnl.load [[PARAM_0_]]{{.}}[[VAR_20_]]#0, [[VAR_20_]]#1] : memref<16x32xi8>
+// CHECK:             [[VAR_22_:%.+]] = arith.extsi [[LOAD_PARAM_0_MEM_]] : i8 to i32
+// CHECK:             krnl.store [[VAR_22_]], [[RES_]]{{.}}[[VAR_20_]]#0, [[VAR_20_]]#1] : memref<16x32xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_1_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_PARAM_2_MEM_:%.+]] = krnl.load [[PARAM_2_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_2_:%.+]] = arith.extsi [[LOAD_PARAM_2_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_2_]], [[RES_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_2_:%.+]] = memref.alloc() {{.*}}: memref<16x32xi32>
+// CHECK-DAG:       [[LOOP_1_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_1_]]#0, [[LOOP_1_]]#1) with ([[LOOP_1_]]#0 -> [[I_2_:%.+]] = 0 to 16, [[LOOP_1_]]#1 -> [[I_3_:%.+]] = 0 to 32){
+// CHECK:             [[VAR_20_1_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_1_]]#0, [[LOOP_1_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_:%.+]] = krnl.load [[RES_]]{{.}}[[VAR_20_1_]]#0, [[VAR_20_1_]]#1] : memref<16x32xi32>
+// CHECK-DAG:         [[VAR_22_1_:%.+]] = krnl.load [[RES_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:             [[VAR_23_:%.+]] = arith.subi [[LOAD_PARAM_0_MEM_1_]], [[VAR_22_1_]] : i32
+// CHECK:             krnl.store [[VAR_23_]], [[RES_2_]]{{.}}[[VAR_20_1_]]#0, [[VAR_20_1_]]#1] : memref<16x32xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_3_:%.+]] = memref.alloc() {{.*}}: memref<32x64xi32>
+// CHECK-DAG:       [[LOOP_2_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_2_]]#0, [[LOOP_2_]]#1) with ([[LOOP_2_]]#0 -> [[I_4_:%.+]] = 0 to 32, [[LOOP_2_]]#1 -> [[I_5_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_20_2_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_2_]]#0, [[LOOP_2_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_:%.+]] = krnl.load [[PARAM_3_]]{{.}}[[VAR_20_2_]]#0, [[VAR_20_2_]]#1] : memref<32x64xi8>
+// CHECK:             [[VAR_22_2_:%.+]] = arith.extsi [[LOAD_PARAM_0_MEM_1_]] : i8 to i32
+// CHECK:             krnl.store [[VAR_22_2_]], [[RES_3_]]{{.}}[[VAR_20_2_]]#0, [[VAR_20_2_]]#1] : memref<32x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_4_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_PARAM_5_MEM_:%.+]] = krnl.load [[PARAM_5_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_6_:%.+]] = arith.extsi [[LOAD_PARAM_5_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_6_]], [[RES_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_5_:%.+]] = memref.alloc() {{.*}}: memref<32x64xi32>
+// CHECK-DAG:       [[LOOP_3_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_3_]]#0, [[LOOP_3_]]#1) with ([[LOOP_3_]]#0 -> [[I_6_:%.+]] = 0 to 32, [[LOOP_3_]]#1 -> [[I_7_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_20_3_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_3_]]#0, [[LOOP_3_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_1_:%.+]] = krnl.load [[RES_3_]]{{.}}[[VAR_20_3_]]#0, [[VAR_20_3_]]#1] : memref<32x64xi32>
+// CHECK-DAG:         [[VAR_22_2_:%.+]] = krnl.load [[RES_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:             [[VAR_23_1_:%.+]] = arith.subi [[LOAD_PARAM_0_MEM_1_1_]], [[VAR_22_2_]] : i32
+// CHECK:             krnl.store [[VAR_23_1_]], [[RES_5_]]{{.}}[[VAR_20_3_]]#0, [[VAR_20_3_]]#1] : memref<32x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_6_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_PARAM_7_MEM_:%.+]] = krnl.load [[PARAM_7_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_9_:%.+]] = arith.extsi [[LOAD_PARAM_7_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_9_]], [[RES_6_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_7_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK-DAG:       [[LOOP_4_:%.+]]:3 = krnl.define_loops 3
+// CHECK:           krnl.iterate([[LOOP_4_]]#0, [[LOOP_4_]]#1) with ([[LOOP_4_]]#0 -> [[I_8_:%.+]] = 0 to 16, [[LOOP_4_]]#1 -> [[I_9_:%.+]] = 0 to 64, [[LOOP_4_]]#2 -> [[I_10_:%.+]] = 0 to 32){
+// CHECK-DAG:         [[VAR_20_4_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_4_]]#0, [[LOOP_4_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_1_:%.+]] = krnl.iterate([[LOOP_4_]]#2) with () iter_args([[VAR_arg11_:%.+]] = [[CST_0_]]) -> (i32){
+// CHECK-DAG:           [[VAR_22_3_:%.+]] = krnl.get_induction_var_value([[LOOP_4_]]#2) : (!krnl.loop) -> index
+// CHECK-NOT: separator of consecutive DAGs
+// CHECK-DAG:           [[VAR_23_1_:%.+]] = krnl.load [[RES_2_]]{{.}}[[VAR_20_4_]]#0, [[VAR_22_3_]]{{.}} : memref<16x32xi32>
+// CHECK-DAG:           [[LOAD_RES_5_MEM_:%.+]] = krnl.load [[RES_5_]]{{.}}[[VAR_22_3_]], [[VAR_20_4_]]#1] : memref<32x64xi32>
+// CHECK:               [[VAR_25_:%.+]] = arith.muli [[VAR_23_1_]], [[LOAD_RES_5_MEM_]] : i32
+// CHECK:               [[VAR_26_:%.+]] = arith.addi [[VAR_arg11_]], [[VAR_25_]] : i32
+// CHECK:               krnl.yield [[VAR_26_]] : i32
+// CHECK:             }
+// CHECK:             krnl.store [[LOAD_PARAM_0_MEM_1_1_]], [[RES_7_]]{{.}}[[VAR_20_4_]]#0, [[VAR_20_4_]]#1] : memref<16x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_8_:%.+]] = memref.alloc() {{.*}}: memref<16x64xf32>
+// CHECK-DAG:       [[LOOP_5_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_5_]]#0, [[LOOP_5_]]#1) with ([[LOOP_5_]]#0 -> [[I_11_:%.+]] = 0 to 16, [[LOOP_5_]]#1 -> [[I_12_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_20_5_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_5_]]#0, [[LOOP_5_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_1_1_:%.+]] = krnl.load [[RES_7_]]{{.}}[[VAR_20_5_]]#0, [[VAR_20_5_]]#1] : memref<16x64xi32>
+// CHECK:             [[VAR_22_4_:%.+]] = arith.sitofp [[LOAD_PARAM_0_MEM_1_1_1_]] : i32 to f32
+// CHECK:             krnl.store [[VAR_22_4_]], [[RES_8_]]{{.}}[[VAR_20_5_]]#0, [[VAR_20_5_]]#1] : memref<16x64xf32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_9_:%.+]] = memref.alloc() {{.*}}: memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_1_MEM_:%.+]] = krnl.load [[PARAM_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_4_MEM_:%.+]] = krnl.load [[PARAM_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:           [[VAR_14_:%.+]] = arith.mulf [[LOAD_PARAM_1_MEM_]], [[LOAD_PARAM_4_MEM_]] : f32
+// CHECK:           krnl.store [[VAR_14_]], [[RES_9_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[RES_10_:%.+]] = memref.alloc() {{.*}}: memref<1xf32>
+// CHECK-DAG:       [[LOAD_RES_9_MEM_:%.+]] = krnl.load [[RES_9_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_6_MEM_:%.+]] = krnl.load [[PARAM_6_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:           [[VAR_17_:%.+]] = arith.divf [[LOAD_RES_9_MEM_]], [[LOAD_PARAM_6_MEM_]] : f32
+// CHECK:           krnl.store [[VAR_17_]], [[RES_10_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[RES_11_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK-DAG:       [[LOOP_6_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_6_]]#0, [[LOOP_6_]]#1) with ([[LOOP_6_]]#0 -> [[I_13_:%.+]] = 0 to 16, [[LOOP_6_]]#1 -> [[I_14_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_20_6_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_6_]]#0, [[LOOP_6_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_1_1_:%.+]] = krnl.load [[RES_8_]]{{.}}[[VAR_20_6_]]#0, [[VAR_20_6_]]#1] : memref<16x64xf32>
+// CHECK-DAG:         [[VAR_22_4_:%.+]] = krnl.load [[RES_10_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:             [[VAR_23_2_:%.+]] = arith.mulf [[LOAD_PARAM_0_MEM_1_1_1_]], [[VAR_22_4_]] : f32
+// CHECK:             [[LOAD_RES_5_MEM_1_:%.+]] = math.floor [[VAR_23_2_]] : f32
+// CHECK:             [[VAR_25_1_:%.+]] = arith.subf [[VAR_23_2_]], [[LOAD_RES_5_MEM_1_]] : f32
+// CHECK-DAG:         [[VAR_26_1_:%.+]] = arith.cmpf ogt, [[VAR_25_1_]], [[CST_5_dot_000000_]] : f32
+// CHECK-DAG:         [[VAR_27_:%.+]] = arith.addf [[LOAD_RES_5_MEM_1_]], [[CST_1_dot_000000_]] : f32
+// CHECK-NOT: separator of consecutive DAGs
+// CHECK-DAG:         [[VAR_28_:%.+]] = arith.select [[VAR_26_1_]], [[VAR_27_]], [[LOAD_RES_5_MEM_1_]] : f32
+// CHECK-DAG:         [[VAR_29_:%.+]] = arith.mulf [[LOAD_RES_5_MEM_1_]], [[CST_5_dot_000000_]] : f32
+// CHECK:             [[VAR_30_:%.+]] = math.floor [[VAR_29_]] : f32
+// CHECK:             [[VAR_31_:%.+]] = arith.mulf [[VAR_30_]], [[CST_2_dot_000000_]] : f32
+// CHECK:             [[VAR_32_:%.+]] = arith.subf [[LOAD_RES_5_MEM_1_]], [[VAR_31_]] : f32
+// CHECK-DAG:         [[VAR_33_:%.+]] = arith.cmpf oeq, [[VAR_32_]], [[CST_1_dot_000000_]] : f32
+// CHECK-DAG:         [[VAR_34_:%.+]] = arith.addf [[LOAD_RES_5_MEM_1_]], [[CST_1_dot_000000_]] : f32
+// CHECK-NOT: separator of consecutive DAGs
+// CHECK-DAG:         [[VAR_35_:%.+]] = arith.select [[VAR_33_]], [[VAR_34_]], [[LOAD_RES_5_MEM_1_]] : f32
+// CHECK-DAG:         [[VAR_36_:%.+]] = arith.cmpf oeq, [[VAR_25_1_]], [[CST_5_dot_000000_]] : f32
+// CHECK:             [[VAR_37_:%.+]] = arith.select [[VAR_36_]], [[VAR_35_]], [[VAR_28_]] : f32
+// CHECK:             [[VAR_38_:%.+]] = arith.fptosi [[VAR_37_]] : f32 to i32
+// CHECK:             krnl.store [[VAR_38_]], [[RES_11_]]{{.}}[[VAR_20_6_]]#0, [[VAR_20_6_]]#1] : memref<16x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_12_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi8>
+// CHECK-DAG:       [[LOOP_7_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_7_]]#0, [[LOOP_7_]]#1) with ([[LOOP_7_]]#0 -> [[I_15_:%.+]] = 0 to 16, [[LOOP_7_]]#1 -> [[I_16_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_20_7_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_7_]]#0, [[LOOP_7_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_1_1_1_:%.+]] = krnl.load [[RES_11_]]{{.}}[[VAR_20_7_]]#0, [[VAR_20_7_]]#1] : memref<16x64xi32>
+// CHECK-DAG:         [[VAR_22_4_1_:%.+]] = krnl.load [[RES_6_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:             [[VAR_23_3_:%.+]] = arith.addi [[LOAD_PARAM_0_MEM_1_1_1_1_]], [[VAR_22_4_1_]] : i32
+// CHECK:             [[LOAD_RES_5_MEM_1_:%.+]] = arith.trunci [[VAR_23_3_]] : i32 to i8
+// CHECK:             krnl.store [[LOAD_RES_5_MEM_1_]], [[RES_12_]]{{.}}[[VAR_20_7_]]#0, [[VAR_20_7_]]#1] : memref<16x64xi8>
+// CHECK:           }
+// CHECK:           return [[RES_12_]] : memref<16x64xi8>
+// CHECK:         }
+}
+
+//-----
+
+func.func @qlinearmatmul_ui8_f32(%arg0: tensor<16x32xui8>, %arg1: tensor<1xf32>, %arg2: tensor<1xui8>, %arg3: tensor<32x64xui8>, %arg4: tensor<1xf32>, %arg5: tensor<1xui8>, %arg6: tensor<1xf32>, %arg7: tensor<1xui8>) -> (tensor<16x64xui8>) {
+    %0 = "onnx.QLinearMatMul"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %arg6, %arg7) : (tensor<16x32xui8>, tensor<1xf32>, tensor<1xui8>, tensor<32x64xui8>, tensor<1xf32>, tensor<1xui8>, tensor<1xf32>, tensor<1xui8>) -> tensor<16x64xui8>
+    return %0 : tensor<16x64xui8>
+
+// CHECK-LABEL:  func.func @qlinearmatmul_ui8_f32
+// CHECK-SAME:   ([[PARAM_0_:%.+]]: memref<16x32xui8>, [[PARAM_1_:%.+]]: memref<1xf32>, [[PARAM_2_:%.+]]: memref<1xui8>, [[PARAM_3_:%.+]]: memref<32x64xui8>, [[PARAM_4_:%.+]]: memref<1xf32>, [[PARAM_5_:%.+]]: memref<1xui8>, [[PARAM_6_:%.+]]: memref<1xf32>, [[PARAM_7_:%.+]]: memref<1xui8>) -> memref<16x64xui8> {
+// CHECK-DAG:       [[CST_5_dot_000000_:%.+]] = arith.constant 5.000000e-01 : f32
+// CHECK-DAG:       [[CST_2_dot_000000_:%.+]] = arith.constant 2.000000e+00 : f32
+// CHECK-DAG:       [[CST_1_dot_000000_:%.+]] = arith.constant 1.000000e+00 : f32
+// CHECK-DAG:       [[CST_0_:%.+]] = arith.constant 0 : i32
+// CHECK-DAG:       [[CST_0_1_:%.+]] = arith.constant 0 : index
+// CHECK-DAG:       [[VAR_0_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i32>} : () -> memref<i32>
+// CHECK-DAG:       [[VAR_1_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_:%.+]] = memref.alloc() {{.*}}: memref<16x32xi32>
+// CHECK-DAG:       [[LOOP_0_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_0_]]#0, [[LOOP_0_]]#1) with ([[LOOP_0_]]#0 -> [[I_0_:%.+]] = 0 to 16, [[LOOP_0_]]#1 -> [[I_1_:%.+]] = 0 to 32){
+// CHECK:             [[VAR_50_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_0_]]#0, [[LOOP_0_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_:%.+]] = krnl.load [[PARAM_0_]]{{.}}[[VAR_50_]]#0, [[VAR_50_]]#1] : memref<16x32xui8>
+// CHECK:             [[VAR_52_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_0_MEM_]] : ui8 to i8
+// CHECK-DAG:         [[VAR_53_:%.+]] = arith.extui [[VAR_52_]] : i8 to i16
+// CHECK-DAG:         [[LOAD_VAR_1_MEM_:%.+]] = krnl.load [[VAR_1_]][] : memref<i16>
+// CHECK:             [[VAR_55_:%.+]] = arith.subi [[VAR_53_]], [[LOAD_VAR_1_MEM_]] : i16
+// CHECK:             [[VAR_56_:%.+]] = arith.trunci [[VAR_55_]] : i16 to i8
+// CHECK:             [[VAR_57_:%.+]] = arith.extsi [[VAR_56_]] : i8 to i32
+// CHECK:             krnl.store [[VAR_57_]], [[RES_]]{{.}}[[VAR_50_]]#0, [[VAR_50_]]#1] : memref<16x32xi32>
+// CHECK:           }
+// CHECK-DAG:       [[VAR_3_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_1_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_PARAM_2_MEM_:%.+]] = krnl.load [[PARAM_2_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xui8>
+// CHECK:           [[VAR_5_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_2_MEM_]] : ui8 to i8
+// CHECK:           [[VAR_6_:%.+]] = arith.extui [[VAR_5_]] : i8 to i16
+// CHECK:           krnl.store [[VAR_6_]], [[RES_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_2_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_RES_1_MEM_:%.+]] = krnl.load [[RES_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[LOAD_VAR_3_MEM_:%.+]] = krnl.load [[VAR_3_]][] : memref<i16>
+// CHECK:           [[VAR_9_:%.+]] = arith.subi [[LOAD_RES_1_MEM_]], [[LOAD_VAR_3_MEM_]] : i16
+// CHECK:           krnl.store [[VAR_9_]], [[RES_2_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_3_:%.+]] = memref.alloc() {{.*}}: memref<1xi8>
+// CHECK-DAG:       [[LOAD_RES_2_MEM_:%.+]] = krnl.load [[RES_2_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK:           [[VAR_11_:%.+]] = arith.trunci [[LOAD_RES_2_MEM_]] : i16 to i8
+// CHECK:           krnl.store [[VAR_11_]], [[RES_3_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK-DAG:       [[RES_4_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_RES_3_MEM_:%.+]] = krnl.load [[RES_3_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_13_:%.+]] = arith.extsi [[LOAD_RES_3_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_13_]], [[RES_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_5_:%.+]] = memref.alloc() {{.*}}: memref<16x32xi32>
+// CHECK-DAG:       [[LOOP_1_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_1_]]#0, [[LOOP_1_]]#1) with ([[LOOP_1_]]#0 -> [[I_2_:%.+]] = 0 to 16, [[LOOP_1_]]#1 -> [[I_3_:%.+]] = 0 to 32){
+// CHECK:             [[VAR_50_1_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_1_]]#0, [[LOOP_1_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_:%.+]] = krnl.load [[RES_]]{{.}}[[VAR_50_1_]]#0, [[VAR_50_1_]]#1] : memref<16x32xi32>
+// CHECK-DAG:         [[VAR_52_1_:%.+]] = krnl.load [[RES_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:             [[VAR_53_1_:%.+]] = arith.subi [[LOAD_PARAM_0_MEM_1_]], [[VAR_52_1_]] : i32
+// CHECK:             krnl.store [[VAR_53_1_]], [[RES_5_]]{{.}}[[VAR_50_1_]]#0, [[VAR_50_1_]]#1] : memref<16x32xi32>
+// CHECK:           }
+// CHECK-DAG:       [[VAR_15_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_6_:%.+]] = memref.alloc() {{.*}}: memref<32x64xi32>
+// CHECK-DAG:       [[LOOP_2_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_2_]]#0, [[LOOP_2_]]#1) with ([[LOOP_2_]]#0 -> [[I_4_:%.+]] = 0 to 32, [[LOOP_2_]]#1 -> [[I_5_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_50_2_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_2_]]#0, [[LOOP_2_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_:%.+]] = krnl.load [[PARAM_3_]]{{.}}[[VAR_50_2_]]#0, [[VAR_50_2_]]#1] : memref<32x64xui8>
+// CHECK:             [[VAR_52_2_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_0_MEM_1_]] : ui8 to i8
+// CHECK-DAG:         [[VAR_53_2_:%.+]] = arith.extui [[VAR_52_2_]] : i8 to i16
+// CHECK-DAG:         [[LOAD_VAR_1_MEM_1_:%.+]] = krnl.load [[VAR_15_]][] : memref<i16>
+// CHECK:             [[VAR_55_1_:%.+]] = arith.subi [[VAR_53_2_]], [[LOAD_VAR_1_MEM_1_]] : i16
+// CHECK:             [[VAR_56_1_:%.+]] = arith.trunci [[VAR_55_1_]] : i16 to i8
+// CHECK:             [[VAR_57_1_:%.+]] = arith.extsi [[VAR_56_1_]] : i8 to i32
+// CHECK:             krnl.store [[VAR_57_1_]], [[RES_6_]]{{.}}[[VAR_50_2_]]#0, [[VAR_50_2_]]#1] : memref<32x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[VAR_17_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_7_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_PARAM_5_MEM_:%.+]] = krnl.load [[PARAM_5_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xui8>
+// CHECK:           [[VAR_19_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_5_MEM_]] : ui8 to i8
+// CHECK:           [[VAR_20_:%.+]] = arith.extui [[VAR_19_]] : i8 to i16
+// CHECK:           krnl.store [[VAR_20_]], [[RES_7_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_8_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_RES_7_MEM_:%.+]] = krnl.load [[RES_7_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[LOAD_VAR_17_MEM_:%.+]] = krnl.load [[VAR_17_]][] : memref<i16>
+// CHECK:           [[VAR_23_:%.+]] = arith.subi [[LOAD_RES_7_MEM_]], [[LOAD_VAR_17_MEM_]] : i16
+// CHECK:           krnl.store [[VAR_23_]], [[RES_8_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_9_:%.+]] = memref.alloc() {{.*}}: memref<1xi8>
+// CHECK-DAG:       [[LOAD_RES_8_MEM_:%.+]] = krnl.load [[RES_8_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK:           [[VAR_25_:%.+]] = arith.trunci [[LOAD_RES_8_MEM_]] : i16 to i8
+// CHECK:           krnl.store [[VAR_25_]], [[RES_9_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK-DAG:       [[RES_10_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_RES_9_MEM_:%.+]] = krnl.load [[RES_9_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_27_:%.+]] = arith.extsi [[LOAD_RES_9_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_27_]], [[RES_10_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_11_:%.+]] = memref.alloc() {{.*}}: memref<32x64xi32>
+// CHECK-DAG:       [[LOOP_3_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_3_]]#0, [[LOOP_3_]]#1) with ([[LOOP_3_]]#0 -> [[I_6_:%.+]] = 0 to 32, [[LOOP_3_]]#1 -> [[I_7_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_50_3_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_3_]]#0, [[LOOP_3_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_1_:%.+]] = krnl.load [[RES_6_]]{{.}}[[VAR_50_3_]]#0, [[VAR_50_3_]]#1] : memref<32x64xi32>
+// CHECK-DAG:         [[VAR_52_2_:%.+]] = krnl.load [[RES_10_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:             [[VAR_53_3_:%.+]] = arith.subi [[LOAD_PARAM_0_MEM_1_1_]], [[VAR_52_2_]] : i32
+// CHECK:             krnl.store [[VAR_53_3_]], [[RES_11_]]{{.}}[[VAR_50_3_]]#0, [[VAR_50_3_]]#1] : memref<32x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[VAR_29_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_12_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_PARAM_7_MEM_:%.+]] = krnl.load [[PARAM_7_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xui8>
+// CHECK:           [[VAR_31_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_7_MEM_]] : ui8 to i8
+// CHECK:           [[VAR_32_:%.+]] = arith.extui [[VAR_31_]] : i8 to i16
+// CHECK:           krnl.store [[VAR_32_]], [[RES_12_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_13_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_RES_12_MEM_:%.+]] = krnl.load [[RES_12_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[LOAD_VAR_29_MEM_:%.+]] = krnl.load [[VAR_29_]][] : memref<i16>
+// CHECK:           [[VAR_35_:%.+]] = arith.subi [[LOAD_RES_12_MEM_]], [[LOAD_VAR_29_MEM_]] : i16
+// CHECK:           krnl.store [[VAR_35_]], [[RES_13_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_14_:%.+]] = memref.alloc() {{.*}}: memref<1xi8>
+// CHECK-DAG:       [[LOAD_RES_13_MEM_:%.+]] = krnl.load [[RES_13_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK:           [[VAR_37_:%.+]] = arith.trunci [[LOAD_RES_13_MEM_]] : i16 to i8
+// CHECK:           krnl.store [[VAR_37_]], [[RES_14_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK-DAG:       [[RES_15_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_RES_14_MEM_:%.+]] = krnl.load [[RES_14_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_39_:%.+]] = arith.extsi [[LOAD_RES_14_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_39_]], [[RES_15_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_16_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK-DAG:       [[LOOP_4_:%.+]]:3 = krnl.define_loops 3
+// CHECK:           krnl.iterate([[LOOP_4_]]#0, [[LOOP_4_]]#1) with ([[LOOP_4_]]#0 -> [[I_8_:%.+]] = 0 to 16, [[LOOP_4_]]#1 -> [[I_9_:%.+]] = 0 to 64, [[LOOP_4_]]#2 -> [[I_10_:%.+]] = 0 to 32){
+// CHECK-DAG:         [[VAR_50_4_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_4_]]#0, [[LOOP_4_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_1_:%.+]] = krnl.iterate([[LOOP_4_]]#2) with () iter_args([[VAR_arg11_:%.+]] = [[CST_0_]]) -> (i32){
+// CHECK-DAG:           [[VAR_52_3_:%.+]] = krnl.get_induction_var_value([[LOOP_4_]]#2) : (!krnl.loop) -> index
+// CHECK-NOT: separator of consecutive DAGs
+// CHECK-DAG:           [[VAR_53_3_:%.+]] = krnl.load [[RES_5_]]{{.}}[[VAR_50_4_]]#0, [[VAR_52_3_]]{{.}} : memref<16x32xi32>
+// CHECK-DAG:           [[LOAD_VAR_1_MEM_1_:%.+]] = krnl.load [[RES_11_]]{{.}}[[VAR_52_3_]], [[VAR_50_4_]]#1] : memref<32x64xi32>
+// CHECK:               [[VAR_55_2_:%.+]] = arith.muli [[VAR_53_3_]], [[LOAD_VAR_1_MEM_1_]] : i32
+// CHECK:               [[VAR_56_2_:%.+]] = arith.addi [[VAR_arg11_]], [[VAR_55_2_]] : i32
+// CHECK:               krnl.yield [[VAR_56_2_]] : i32
+// CHECK:             }
+// CHECK:             krnl.store [[LOAD_PARAM_0_MEM_1_1_]], [[RES_16_]]{{.}}[[VAR_50_4_]]#0, [[VAR_50_4_]]#1] : memref<16x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_17_:%.+]] = memref.alloc() {{.*}}: memref<16x64xf32>
+// CHECK-DAG:       [[LOOP_5_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_5_]]#0, [[LOOP_5_]]#1) with ([[LOOP_5_]]#0 -> [[I_11_:%.+]] = 0 to 16, [[LOOP_5_]]#1 -> [[I_12_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_50_5_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_5_]]#0, [[LOOP_5_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_1_1_:%.+]] = krnl.load [[RES_16_]]{{.}}[[VAR_50_5_]]#0, [[VAR_50_5_]]#1] : memref<16x64xi32>
+// CHECK:             [[VAR_52_4_:%.+]] = arith.sitofp [[LOAD_PARAM_0_MEM_1_1_1_]] : i32 to f32
+// CHECK:             krnl.store [[VAR_52_4_]], [[RES_17_]]{{.}}[[VAR_50_5_]]#0, [[VAR_50_5_]]#1] : memref<16x64xf32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_18_:%.+]] = memref.alloc() {{.*}}: memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_1_MEM_:%.+]] = krnl.load [[PARAM_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_4_MEM_:%.+]] = krnl.load [[PARAM_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:           [[VAR_44_:%.+]] = arith.mulf [[LOAD_PARAM_1_MEM_]], [[LOAD_PARAM_4_MEM_]] : f32
+// CHECK:           krnl.store [[VAR_44_]], [[RES_18_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[RES_19_:%.+]] = memref.alloc() {{.*}}: memref<1xf32>
+// CHECK-DAG:       [[LOAD_RES_18_MEM_:%.+]] = krnl.load [[RES_18_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_6_MEM_:%.+]] = krnl.load [[PARAM_6_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:           [[VAR_47_:%.+]] = arith.divf [[LOAD_RES_18_MEM_]], [[LOAD_PARAM_6_MEM_]] : f32
+// CHECK:           krnl.store [[VAR_47_]], [[RES_19_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[RES_20_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK-DAG:       [[LOOP_6_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_6_]]#0, [[LOOP_6_]]#1) with ([[LOOP_6_]]#0 -> [[I_13_:%.+]] = 0 to 16, [[LOOP_6_]]#1 -> [[I_14_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_50_6_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_6_]]#0, [[LOOP_6_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_1_1_:%.+]] = krnl.load [[RES_17_]]{{.}}[[VAR_50_6_]]#0, [[VAR_50_6_]]#1] : memref<16x64xf32>
+// CHECK-DAG:         [[VAR_52_4_:%.+]] = krnl.load [[RES_19_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:             [[VAR_53_4_:%.+]] = arith.mulf [[LOAD_PARAM_0_MEM_1_1_1_]], [[VAR_52_4_]] : f32
+// CHECK:             [[LOAD_VAR_1_MEM_1_1_:%.+]] = math.floor [[VAR_53_4_]] : f32
+// CHECK:             [[VAR_55_3_:%.+]] = arith.subf [[VAR_53_4_]], [[LOAD_VAR_1_MEM_1_1_]] : f32
+// CHECK-DAG:         [[VAR_56_3_:%.+]] = arith.cmpf ogt, [[VAR_55_3_]], [[CST_5_dot_000000_]] : f32
+// CHECK-DAG:         [[VAR_57_2_:%.+]] = arith.addf [[LOAD_VAR_1_MEM_1_1_]], [[CST_1_dot_000000_]] : f32
+// CHECK-NOT: separator of consecutive DAGs
+// CHECK-DAG:         [[VAR_58_:%.+]] = arith.select [[VAR_56_3_]], [[VAR_57_2_]], [[LOAD_VAR_1_MEM_1_1_]] : f32
+// CHECK-DAG:         [[VAR_59_:%.+]] = arith.mulf [[LOAD_VAR_1_MEM_1_1_]], [[CST_5_dot_000000_]] : f32
+// CHECK:             [[VAR_60_:%.+]] = math.floor [[VAR_59_]] : f32
+// CHECK:             [[VAR_61_:%.+]] = arith.mulf [[VAR_60_]], [[CST_2_dot_000000_]] : f32
+// CHECK:             [[VAR_62_:%.+]] = arith.subf [[LOAD_VAR_1_MEM_1_1_]], [[VAR_61_]] : f32
+// CHECK-DAG:         [[VAR_63_:%.+]] = arith.cmpf oeq, [[VAR_62_]], [[CST_1_dot_000000_]] : f32
+// CHECK-DAG:         [[VAR_64_:%.+]] = arith.addf [[LOAD_VAR_1_MEM_1_1_]], [[CST_1_dot_000000_]] : f32
+// CHECK-NOT: separator of consecutive DAGs
+// CHECK-DAG:         [[VAR_65_:%.+]] = arith.select [[VAR_63_]], [[VAR_64_]], [[LOAD_VAR_1_MEM_1_1_]] : f32
+// CHECK-DAG:         [[VAR_66_:%.+]] = arith.cmpf oeq, [[VAR_55_3_]], [[CST_5_dot_000000_]] : f32
+// CHECK:             [[VAR_67_:%.+]] = arith.select [[VAR_66_]], [[VAR_65_]], [[VAR_58_]] : f32
+// CHECK:             [[VAR_68_:%.+]] = arith.fptosi [[VAR_67_]] : f32 to i32
+// CHECK:             krnl.store [[VAR_68_]], [[RES_20_]]{{.}}[[VAR_50_6_]]#0, [[VAR_50_6_]]#1] : memref<16x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_21_:%.+]] = memref.alloc() {{.*}}: memref<16x64xui8>
+// CHECK-DAG:       [[LOOP_7_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_7_]]#0, [[LOOP_7_]]#1) with ([[LOOP_7_]]#0 -> [[I_15_:%.+]] = 0 to 16, [[LOOP_7_]]#1 -> [[I_16_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_50_7_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_7_]]#0, [[LOOP_7_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK-DAG:         [[LOAD_PARAM_0_MEM_1_1_1_1_:%.+]] = krnl.load [[RES_20_]]{{.}}[[VAR_50_7_]]#0, [[VAR_50_7_]]#1] : memref<16x64xi32>
+// CHECK-DAG:         [[VAR_52_4_1_:%.+]] = krnl.load [[RES_15_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-NOT: separator of consecutive DAGs
+// CHECK-DAG:         [[VAR_53_5_:%.+]] = arith.addi [[LOAD_PARAM_0_MEM_1_1_1_1_]], [[VAR_52_4_1_]] : i32
+// CHECK-DAG:         [[LOAD_VAR_1_MEM_1_1_:%.+]] = krnl.load [[VAR_0_]][] : memref<i32>
+// CHECK:             [[VAR_55_4_:%.+]] = arith.addi [[VAR_53_5_]], [[LOAD_VAR_1_MEM_1_1_]] : i32
+// CHECK:             [[VAR_56_4_:%.+]] = arith.trunci [[VAR_55_4_]] : i32 to i8
+// CHECK:             [[VAR_57_3_:%.+]] = builtin.unrealized_conversion_cast [[VAR_56_4_]] : i8 to ui8
+// CHECK:             krnl.store [[VAR_57_3_]], [[RES_21_]]{{.}}[[VAR_50_7_]]#0, [[VAR_50_7_]]#1] : memref<16x64xui8>
+// CHECK:           }
+// CHECK:           return [[RES_21_]] : memref<16x64xui8>
+// CHECK:         }
+}
diff --git a/test/mlir/conversion/onnx_to_krnl/Math/QLinearMatMul_with_canonicalize_O3.mlir b/test/mlir/conversion/onnx_to_krnl/Math/QLinearMatMul_with_canonicalize_O3.mlir
new file mode 100644
index 00000000000..b895f57fa29
--- /dev/null
+++ b/test/mlir/conversion/onnx_to_krnl/Math/QLinearMatMul_with_canonicalize_O3.mlir
@@ -0,0 +1,500 @@
+// RUN: onnx-mlir-opt -O3 --mtriple=s390x-ibm-loz --march=z16 --shape-inference --convert-onnx-to-krnl --canonicalize %s -split-input-file | FileCheck %s
+
+func.func @qlinearmatmul_i8_f32(%arg0: tensor<16x32xi8>, %arg1: tensor<1xf32>, %arg2: tensor<1xi8>, %arg3: tensor<32x64xi8>, %arg4: tensor<1xf32>, %arg5: tensor<1xi8>, %arg6: tensor<1xf32>, %arg7: tensor<1xi8>) -> (tensor<16x64xi8>) {
+    %0 = "onnx.QLinearMatMul"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %arg6, %arg7) : (tensor<16x32xi8>, tensor<1xf32>, tensor<1xi8>, tensor<32x64xi8>, tensor<1xf32>, tensor<1xi8>, tensor<1xf32>, tensor<1xi8>) -> tensor<16x64xi8>
+    return %0 : tensor<16x64xi8>
+
+// CHECK-LABEL:  func.func @qlinearmatmul_i8_f32
+// CHECK-SAME:   ([[PARAM_0_:%.+]]: memref<16x32xi8>, [[PARAM_1_:%.+]]: memref<1xf32>, [[PARAM_2_:%.+]]: memref<1xi8>, [[PARAM_3_:%.+]]: memref<32x64xi8>, [[PARAM_4_:%.+]]: memref<1xf32>, [[PARAM_5_:%.+]]: memref<1xi8>, [[PARAM_6_:%.+]]: memref<1xf32>, [[PARAM_7_:%.+]]: memref<1xi8>) -> memref<16x64xi8> {
+// CHECK-DAG:       [[CST_1024_:%.+]] = arith.constant 1024 : index
+// CHECK-DAG:       [[CST_0_:%.+]] = arith.constant 0 : i32
+// CHECK-DAG:       [[CST_2048_:%.+]] = arith.constant 2048 : index
+// CHECK-DAG:       [[CST_512_:%.+]] = arith.constant 512 : index
+// CHECK-DAG:       [[CST_0_1_:%.+]] = arith.constant 0 : index
+// CHECK-DAG:       [[CST_16_:%.+]] = arith.constant 16 : index
+// CHECK-DAG:       [[CST_32_:%.+]] = arith.constant 32 : index
+// CHECK-DAG:       [[CST_64_:%.+]] = arith.constant 64 : index
+// CHECK-DAG:       [[RES_:%.+]] = memref.alloc() {{.*}}: memref<16x32xi32>
+// CHECK-DAG:       [[LOOP_0_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_0_]]#0, [[LOOP_0_]]#1) with ([[LOOP_0_]]#0 -> [[I_0_:%.+]] = 0 to 16, [[LOOP_0_]]#1 -> [[I_1_:%.+]] = 0 to 32){
+// CHECK:             [[VAR_18_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_0_]]#0, [[LOOP_0_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_:%.+]] = krnl.load [[PARAM_0_]]{{.}}[[VAR_18_]]#0, [[VAR_18_]]#1] : memref<16x32xi8>
+// CHECK:             [[VAR_20_:%.+]] = arith.extsi [[LOAD_PARAM_0_MEM_]] : i8 to i32
+// CHECK:             krnl.store [[VAR_20_]], [[RES_]]{{.}}[[VAR_18_]]#0, [[VAR_18_]]#1] : memref<16x32xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_1_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_PARAM_2_MEM_:%.+]] = krnl.load [[PARAM_2_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_2_:%.+]] = arith.extsi [[LOAD_PARAM_2_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_2_]], [[RES_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_2_:%.+]] = memref.alloc() {{.*}}: memref<16x32xi32>
+// CHECK-DAG:       [[RES_3_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_512_]], [[RES_3_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_:%.+]] = memref.reshape [[RES_]]([[RES_]]_2) : (memref<16x32xi32>, memref<1xindex>) -> memref<512xi32>
+// CHECK-DAG:       [[RES_4_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_512_]], [[RES_4_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_4_:%.+]] = memref.reshape [[RES_2_]]([[RES_4_]]) : (memref<16x32xi32>, memref<1xindex>) -> memref<512xi32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_1_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__0_:%.+]], [[BLOCK_IN__0_:%.+]] = krnl.block [[LOOP_1_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__0_]]) with ([[LOOP_1_]] -> [[I_2_:%.+]] = 0 to 512){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__0_]]) : (!krnl.loop) -> index
+// CHECK-DAG:           [[VAR_20_1_:%.+]] = vector.load [[VAR_reshape_]]{{.}}[[LOAD_PARAM_0_MEM_1_]]{{.}} : memref<512xi32>, vector<32xi32>
+// CHECK-DAG:           [[LOAD_RES_1_MEM_:%.+]] = krnl.load [[RES_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:               [[VAR_22_:%.+]] = vector.splat [[LOAD_RES_1_MEM_]] : vector<32xi32>
+// CHECK:               [[VAR_23_:%.+]] = arith.subi [[VAR_20_1_]], [[VAR_22_]] : vector<32xi32>
+// CHECK:               vector.store [[VAR_23_]], [[VAR_reshape_4_]]{{.}}[[LOAD_PARAM_0_MEM_1_]]{{.}} : memref<512xi32>, vector<32xi32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[RES_5_:%.+]] = memref.alloc() {{.*}}: memref<32x64xi32>
+// CHECK-DAG:       [[LOOP_2_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_2_]]#0, [[LOOP_2_]]#1) with ([[LOOP_2_]]#0 -> [[I_3_:%.+]] = 0 to 32, [[LOOP_2_]]#1 -> [[I_4_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_18_1_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_2_]]#0, [[LOOP_2_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_:%.+]] = krnl.load [[PARAM_3_]]{{.}}[[VAR_18_1_]]#0, [[VAR_18_1_]]#1] : memref<32x64xi8>
+// CHECK:             [[VAR_20_2_:%.+]] = arith.extsi [[LOAD_PARAM_0_MEM_1_]] : i8 to i32
+// CHECK:             krnl.store [[VAR_20_2_]], [[RES_5_]]{{.}}[[VAR_18_1_]]#0, [[VAR_18_1_]]#1] : memref<32x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_6_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_PARAM_5_MEM_:%.+]] = krnl.load [[PARAM_5_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_5_:%.+]] = arith.extsi [[LOAD_PARAM_5_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_5_]], [[RES_6_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_7_:%.+]] = memref.alloc() {{.*}}: memref<32x64xi32>
+// CHECK-DAG:       [[RES_8_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_2048_]], [[RES_8_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_9_:%.+]] = memref.reshape [[RES_5_]]([[RES_8_]]) : (memref<32x64xi32>, memref<1xindex>) -> memref<2048xi32>
+// CHECK-DAG:       [[RES_9_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_2048_]], [[RES_9_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_11_:%.+]] = memref.reshape [[RES_7_]]([[RES_9_]]) : (memref<32x64xi32>, memref<1xindex>) -> memref<2048xi32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_3_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__1_:%.+]], [[BLOCK_IN__1_:%.+]] = krnl.block [[LOOP_3_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__1_]]) with ([[LOOP_3_]] -> [[I_5_:%.+]] = 0 to 2048){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__1_]]) : (!krnl.loop) -> index
+// CHECK-DAG:           [[VAR_20_2_:%.+]] = vector.load [[VAR_reshape_9_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_]]{{.}} : memref<2048xi32>, vector<32xi32>
+// CHECK-DAG:           [[LOAD_RES_1_MEM_1_:%.+]] = krnl.load [[RES_6_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:               [[VAR_22_1_:%.+]] = vector.splat [[LOAD_RES_1_MEM_1_]] : vector<32xi32>
+// CHECK:               [[VAR_23_1_:%.+]] = arith.subi [[VAR_20_2_]], [[VAR_22_1_]] : vector<32xi32>
+// CHECK:               vector.store [[VAR_23_1_]], [[VAR_reshape_11_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_]]{{.}} : memref<2048xi32>, vector<32xi32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[RES_10_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_PARAM_7_MEM_:%.+]] = krnl.load [[PARAM_7_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_7_:%.+]] = arith.extsi [[LOAD_PARAM_7_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_7_]], [[RES_10_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:           [[RES_11_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK:           krnl.memset [[RES_11_]], [[CST_0_]] : memref<16x64xi32>
+// CHECK:           [[LOOP_4_:%.+]]:3 = krnl.define_loops 3
+// CHECK:           [[BLOCK_TILE__2_:%.+]], [[BLOCK_IN__2_:%.+]] = krnl.block [[LOOP_4_]]#0 4 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:           [[BLOCK_TILE__3_:%.+]], [[BLOCK_IN__3_:%.+]] = krnl.block [[LOOP_4_]]#1 8 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:           [[BLOCK_TILE__4_:%.+]], [[BLOCK_IN__4_:%.+]] = krnl.block [[LOOP_4_]]#2 8 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:           krnl.permute([[BLOCK_TILE__2_]], [[BLOCK_IN__2_]], [[BLOCK_TILE__2_]]_14, [[BLOCK_IN__2_]]_15, [[BLOCK_TILE__2_]]_16, [[BLOCK_IN__2_]]_17) [0, 3, 1, 4, 2, 5] : !krnl.loop, !krnl.loop, !krnl.loop, !krnl.loop, !krnl.loop, !krnl.loop
+// CHECK:           krnl.iterate([[BLOCK_TILE__2_]], [[BLOCK_TILE__2_]]_14, [[BLOCK_TILE__2_]]_16) with ([[LOOP_4_]]#0 -> [[I_6_:%.+]] = [[CST_0_1_]] to [[CST_16_]], [[LOOP_4_]]#1 -> [[I_7_:%.+]] = [[CST_0_1_]] to [[CST_64_]], [[LOOP_4_]]#2 -> [[I_8_:%.+]] = [[CST_0_1_]] to [[CST_32_]]){
+// CHECK:             [[VAR_18_2_:%.+]]:3 = krnl.get_induction_var_value([[BLOCK_TILE__2_]], [[BLOCK_TILE__2_]]_14, [[BLOCK_TILE__2_]]_16) : (!krnl.loop, !krnl.loop, !krnl.loop) -> (index, index, index)
+// CHECK:             krnl.matmul [[RES_2_]]{{.}}[[CST_0_1_]], [[CST_0_1_]]{{.}}, [[RES_7_]]{{.}}[[CST_0_1_]], [[CST_0_1_]]{{.}}, [[RES_2_]]3{{.}}[[CST_0_1_]], [[CST_0_1_]]{{.}}, ([[BLOCK_IN__2_]], [[BLOCK_IN__2_]]_15, [[BLOCK_IN__2_]]_17), ([[VAR_18_2_]]#0, [[VAR_18_2_]]#1, [[VAR_18_2_]]#2), ([[CST_16_]], [[CST_64_]], [[CST_32_]]) {aTileSize = [], bTileSize = [], cTileSize = [], computeTileSize = [4, 8, 8]} : memref<16x32xi32>, memref<32x64xi32>, memref<16x64xi32>, (!krnl.loop, !krnl.loop, !krnl.loop)
+// CHECK:           }
+// CHECK-DAG:       [[RES_12_:%.+]] = memref.alloc() {{.*}}: memref<16x64xf32>
+// CHECK-DAG:       [[LOOP_5_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_5_]]#0, [[LOOP_5_]]#1) with ([[LOOP_5_]]#0 -> [[I_9_:%.+]] = 0 to 16, [[LOOP_5_]]#1 -> [[I_10_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_18_3_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_5_]]#0, [[LOOP_5_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_1_:%.+]] = krnl.load [[RES_11_]]{{.}}[[VAR_18_3_]]#0, [[VAR_18_3_]]#1] : memref<16x64xi32>
+// CHECK:             [[VAR_20_3_:%.+]] = arith.sitofp [[LOAD_PARAM_0_MEM_1_1_]] : i32 to f32
+// CHECK:             krnl.store [[VAR_20_3_]], [[RES_12_]]{{.}}[[VAR_18_3_]]#0, [[VAR_18_3_]]#1] : memref<16x64xf32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_13_:%.+]] = memref.alloc() {{.*}}: memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_1_MEM_:%.+]] = krnl.load [[PARAM_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_4_MEM_:%.+]] = krnl.load [[PARAM_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:           [[VAR_12_:%.+]] = arith.mulf [[LOAD_PARAM_1_MEM_]], [[LOAD_PARAM_4_MEM_]] : f32
+// CHECK:           krnl.store [[VAR_12_]], [[RES_13_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[RES_14_:%.+]] = memref.alloc() {{.*}}: memref<1xf32>
+// CHECK-DAG:       [[LOAD_RES_13_MEM_:%.+]] = krnl.load [[RES_13_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_6_MEM_:%.+]] = krnl.load [[PARAM_6_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:           [[VAR_15_:%.+]] = arith.divf [[LOAD_RES_13_MEM_]], [[LOAD_PARAM_6_MEM_]] : f32
+// CHECK:           krnl.store [[VAR_15_]], [[RES_14_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[RES_15_:%.+]] = memref.alloc() {{.*}}: memref<16x64xf32>
+// CHECK-DAG:       [[RES_16_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_16_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_23_:%.+]] = memref.reshape [[RES_12_]]([[RES_16_]]) : (memref<16x64xf32>, memref<1xindex>) -> memref<1024xf32>
+// CHECK-DAG:       [[RES_17_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_17_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_25_:%.+]] = memref.reshape [[RES_15_]]([[RES_17_]]) : (memref<16x64xf32>, memref<1xindex>) -> memref<1024xf32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_6_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__5_:%.+]], [[BLOCK_IN__5_:%.+]] = krnl.block [[LOOP_6_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__5_]]) with ([[LOOP_6_]] -> [[I_11_:%.+]] = 0 to 1024){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_1_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__5_]]) : (!krnl.loop) -> index
+// CHECK-DAG:           [[VAR_20_3_:%.+]] = vector.load [[VAR_reshape_23_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_]]{{.}} : memref<1024xf32>, vector<32xf32>
+// CHECK-DAG:           [[LOAD_RES_1_MEM_1_:%.+]] = krnl.load [[RES_14_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:               [[VAR_22_2_:%.+]] = vector.splat [[LOAD_RES_1_MEM_1_]] : vector<32xf32>
+// CHECK:               [[VAR_23_2_:%.+]] = arith.mulf [[VAR_20_3_]], [[VAR_22_2_]] : vector<32xf32>
+// CHECK:               vector.store [[VAR_23_2_]], [[VAR_reshape_25_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_]]{{.}} : memref<1024xf32>, vector<32xf32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[RES_18_:%.+]] = memref.alloc() {{.*}}: memref<16x64xf32>
+// CHECK-DAG:       [[RES_19_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_19_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_28_:%.+]] = memref.reshape [[RES_15_]]([[RES_19_]]) : (memref<16x64xf32>, memref<1xindex>) -> memref<1024xf32>
+// CHECK-DAG:       [[RES_20_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_20_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_30_:%.+]] = memref.reshape [[RES_18_]]([[RES_20_]]) : (memref<16x64xf32>, memref<1xindex>) -> memref<1024xf32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_7_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__6_:%.+]], [[BLOCK_IN__6_:%.+]] = krnl.block [[LOOP_7_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__6_]]) with ([[LOOP_7_]] -> [[I_12_:%.+]] = 0 to 1024){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_1_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__6_]]) : (!krnl.loop) -> index
+// CHECK:               [[VAR_20_3_1_:%.+]] = vector.load [[VAR_reshape_28_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_]]{{.}} : memref<1024xf32>, vector<32xf32>
+// CHECK:               [[LOAD_RES_1_MEM_1_1_:%.+]] = vector.shape_cast [[VAR_20_3_1_]] : vector<32xf32> to vector<8x4xf32>
+// CHECK:               [[VAR_22_3_:%.+]] = vector.extract [[LOAD_RES_1_MEM_1_1_]][0] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_23_3_:%.+]] = "krnl.round_even"([[VAR_22_3_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_24_:%.+]] = vector.insert [[VAR_23_3_]], [[LOAD_RES_1_MEM_1_1_]] [0] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_25_:%.+]] = vector.extract [[LOAD_RES_1_MEM_1_1_]][1] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_26_:%.+]] = "krnl.round_even"([[VAR_25_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_27_:%.+]] = vector.insert [[VAR_26_]], [[VAR_24_]] [1] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_28_:%.+]] = vector.extract [[LOAD_RES_1_MEM_1_1_]][2] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_29_:%.+]] = "krnl.round_even"([[VAR_28_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_30_:%.+]] = vector.insert [[VAR_29_]], [[VAR_27_]] [2] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_31_:%.+]] = vector.extract [[LOAD_RES_1_MEM_1_1_]][3] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_32_:%.+]] = "krnl.round_even"([[VAR_31_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_33_:%.+]] = vector.insert [[VAR_32_]], [[VAR_30_]] [3] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_34_:%.+]] = vector.extract [[LOAD_RES_1_MEM_1_1_]][4] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_35_:%.+]] = "krnl.round_even"([[VAR_34_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_36_:%.+]] = vector.insert [[VAR_35_]], [[VAR_33_]] [4] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_37_:%.+]] = vector.extract [[LOAD_RES_1_MEM_1_1_]][5] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_38_:%.+]] = "krnl.round_even"([[VAR_37_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_39_:%.+]] = vector.insert [[VAR_38_]], [[VAR_36_]] [5] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_40_:%.+]] = vector.extract [[LOAD_RES_1_MEM_1_1_]][6] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_41_:%.+]] = "krnl.round_even"([[VAR_40_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_42_:%.+]] = vector.insert [[VAR_41_]], [[VAR_39_]] [6] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_43_:%.+]] = vector.extract [[LOAD_RES_1_MEM_1_1_]][7] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_44_:%.+]] = "krnl.round_even"([[VAR_43_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK:               [[VAR_45_:%.+]] = vector.insert [[VAR_44_]], [[VAR_42_]] [7] : vector<4xf32> into vector<8x4xf32>
+// CHECK:               [[VAR_46_:%.+]] = vector.shape_cast [[VAR_45_]] : vector<8x4xf32> to vector<32xf32>
+// CHECK:               vector.store [[VAR_46_]], [[VAR_reshape_30_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_]]{{.}} : memref<1024xf32>, vector<32xf32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[RES_21_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK-DAG:       [[LOOP_8_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_8_]]#0, [[LOOP_8_]]#1) with ([[LOOP_8_]]#0 -> [[I_13_:%.+]] = 0 to 16, [[LOOP_8_]]#1 -> [[I_14_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_18_4_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_8_]]#0, [[LOOP_8_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_1_1_1_:%.+]] = krnl.load [[RES_18_]]{{.}}[[VAR_18_4_]]#0, [[VAR_18_4_]]#1] : memref<16x64xf32>
+// CHECK:             [[VAR_20_4_:%.+]] = arith.fptosi [[LOAD_PARAM_0_MEM_1_1_1_1_]] : f32 to i32
+// CHECK:             krnl.store [[VAR_20_4_]], [[RES_21_]]{{.}}[[VAR_18_4_]]#0, [[VAR_18_4_]]#1] : memref<16x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_22_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK-DAG:       [[RES_23_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_23_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_34_:%.+]] = memref.reshape [[RES_21_]]([[RES_23_]]) : (memref<16x64xi32>, memref<1xindex>) -> memref<1024xi32>
+// CHECK-DAG:       [[RES_24_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_24_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_36_:%.+]] = memref.reshape [[RES_22_]]([[RES_24_]]) : (memref<16x64xi32>, memref<1xindex>) -> memref<1024xi32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_9_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__7_:%.+]], [[BLOCK_IN__7_:%.+]] = krnl.block [[LOOP_9_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__7_]]) with ([[LOOP_9_]] -> [[I_15_:%.+]] = 0 to 1024){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_1_1_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__7_]]) : (!krnl.loop) -> index
+// CHECK-DAG:           [[VAR_20_4_:%.+]] = vector.load [[VAR_reshape_34_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_1_]]{{.}} : memref<1024xi32>, vector<32xi32>
+// CHECK-DAG:           [[LOAD_RES_1_MEM_1_1_:%.+]] = krnl.load [[RES_10_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:               [[VAR_22_4_:%.+]] = vector.splat [[LOAD_RES_1_MEM_1_1_]] : vector<32xi32>
+// CHECK:               [[VAR_23_4_:%.+]] = arith.addi [[VAR_20_4_]], [[VAR_22_4_]] : vector<32xi32>
+// CHECK:               vector.store [[VAR_23_4_]], [[VAR_reshape_36_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_1_]]{{.}} : memref<1024xi32>, vector<32xi32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[RES_25_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi8>
+// CHECK-DAG:       [[LOOP_10_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_10_]]#0, [[LOOP_10_]]#1) with ([[LOOP_10_]]#0 -> [[I_16_:%.+]] = 0 to 16, [[LOOP_10_]]#1 -> [[I_17_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_18_5_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_10_]]#0, [[LOOP_10_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_1_1_1_1_:%.+]] = krnl.load [[RES_22_]]{{.}}[[VAR_18_5_]]#0, [[VAR_18_5_]]#1] : memref<16x64xi32>
+// CHECK:             [[VAR_20_5_:%.+]] = arith.trunci [[LOAD_PARAM_0_MEM_1_1_1_1_1_]] : i32 to i8
+// CHECK:             krnl.store [[VAR_20_5_]], [[RES_25_]]{{.}}[[VAR_18_5_]]#0, [[VAR_18_5_]]#1] : memref<16x64xi8>
+// CHECK:           }
+// CHECK:           return [[RES_25_]] : memref<16x64xi8>
+// CHECK:         }
+}
+
+//-----
+
+func.func @qlinearmatmul_ui8_f32(%arg0: tensor<16x32xui8>, %arg1: tensor<1xf32>, %arg2: tensor<1xui8>, %arg3: tensor<32x64xui8>, %arg4: tensor<1xf32>, %arg5: tensor<1xui8>, %arg6: tensor<1xf32>, %arg7: tensor<1xui8>) -> (tensor<16x64xui8>) {
+    %0 = "onnx.QLinearMatMul"(%arg0, %arg1, %arg2, %arg3, %arg4, %arg5, %arg6, %arg7) : (tensor<16x32xui8>, tensor<1xf32>, tensor<1xui8>, tensor<32x64xui8>, tensor<1xf32>, tensor<1xui8>, tensor<1xf32>, tensor<1xui8>) -> tensor<16x64xui8>
+    return %0 : tensor<16x64xui8>
+
+// CHECK-LABEL:  func.func @qlinearmatmul_ui8_f32
+// CHECK-SAME:   ([[PARAM_0_:%.+]]: memref<16x32xui8>, [[PARAM_1_:%.+]]: memref<1xf32>, [[PARAM_2_:%.+]]: memref<1xui8>, [[PARAM_3_:%.+]]: memref<32x64xui8>, [[PARAM_4_:%.+]]: memref<1xf32>, [[PARAM_5_:%.+]]: memref<1xui8>, [[PARAM_6_:%.+]]: memref<1xf32>, [[PARAM_7_:%.+]]: memref<1xui8>) -> memref<16x64xui8> {
+// CHECK-DAG:       [[CST_1024_:%.+]] = arith.constant 1024 : index
+// CHECK-DAG:       [[CST_0_:%.+]] = arith.constant 0 : i32
+// CHECK-DAG:       [[CST_2048_:%.+]] = arith.constant 2048 : index
+// CHECK-DAG:       [[CST_512_:%.+]] = arith.constant 512 : index
+// CHECK-DAG:       [[CST_0_1_:%.+]] = arith.constant 0 : index
+// CHECK-DAG:       [[CST_16_:%.+]] = arith.constant 16 : index
+// CHECK-DAG:       [[CST_32_:%.+]] = arith.constant 32 : index
+// CHECK-DAG:       [[CST_64_:%.+]] = arith.constant 64 : index
+// CHECK-DAG:       [[VAR_0_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i32>} : () -> memref<i32>
+// CHECK-DAG:       [[VAR_1_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_:%.+]] = memref.alloc() {{.*}}: memref<16x32xi32>
+// CHECK-DAG:       [[LOOP_0_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_0_]]#0, [[LOOP_0_]]#1) with ([[LOOP_0_]]#0 -> [[I_0_:%.+]] = 0 to 16, [[LOOP_0_]]#1 -> [[I_1_:%.+]] = 0 to 32){
+// CHECK:             [[VAR_48_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_0_]]#0, [[LOOP_0_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_:%.+]] = krnl.load [[PARAM_0_]]{{.}}[[VAR_48_]]#0, [[VAR_48_]]#1] : memref<16x32xui8>
+// CHECK:             [[VAR_50_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_0_MEM_]] : ui8 to i8
+// CHECK-DAG:         [[VAR_51_:%.+]] = arith.extui [[VAR_50_]] : i8 to i16
+// CHECK-DAG:         [[LOAD_VAR_1_MEM_:%.+]] = krnl.load [[VAR_1_]][] : memref<i16>
+// CHECK:             [[VAR_53_:%.+]] = arith.subi [[VAR_51_]], [[LOAD_VAR_1_MEM_]] : i16
+// CHECK:             [[VAR_54_:%.+]] = arith.trunci [[VAR_53_]] : i16 to i8
+// CHECK:             [[VAR_55_:%.+]] = arith.extsi [[VAR_54_]] : i8 to i32
+// CHECK:             krnl.store [[VAR_55_]], [[RES_]]{{.}}[[VAR_48_]]#0, [[VAR_48_]]#1] : memref<16x32xi32>
+// CHECK:           }
+// CHECK-DAG:       [[VAR_3_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_1_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_PARAM_2_MEM_:%.+]] = krnl.load [[PARAM_2_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xui8>
+// CHECK:           [[VAR_5_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_2_MEM_]] : ui8 to i8
+// CHECK:           [[VAR_6_:%.+]] = arith.extui [[VAR_5_]] : i8 to i16
+// CHECK:           krnl.store [[VAR_6_]], [[RES_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_2_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_RES_1_MEM_:%.+]] = krnl.load [[RES_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[LOAD_VAR_3_MEM_:%.+]] = krnl.load [[VAR_3_]][] : memref<i16>
+// CHECK:           [[VAR_9_:%.+]] = arith.subi [[LOAD_RES_1_MEM_]], [[LOAD_VAR_3_MEM_]] : i16
+// CHECK:           krnl.store [[VAR_9_]], [[RES_2_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_3_:%.+]] = memref.alloc() {{.*}}: memref<1xi8>
+// CHECK-DAG:       [[LOAD_RES_2_MEM_:%.+]] = krnl.load [[RES_2_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK:           [[VAR_11_:%.+]] = arith.trunci [[LOAD_RES_2_MEM_]] : i16 to i8
+// CHECK:           krnl.store [[VAR_11_]], [[RES_3_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK-DAG:       [[RES_4_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_RES_3_MEM_:%.+]] = krnl.load [[RES_3_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_13_:%.+]] = arith.extsi [[LOAD_RES_3_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_13_]], [[RES_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_5_:%.+]] = memref.alloc() {{.*}}: memref<16x32xi32>
+// CHECK-DAG:       [[RES_6_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_512_]], [[RES_6_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_:%.+]] = memref.reshape [[RES_]]([[RES_]]_5) : (memref<16x32xi32>, memref<1xindex>) -> memref<512xi32>
+// CHECK-DAG:       [[RES_7_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_512_]], [[RES_7_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_7_:%.+]] = memref.reshape [[RES_5_]]([[RES_7_]]) : (memref<16x32xi32>, memref<1xindex>) -> memref<512xi32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_1_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__0_:%.+]], [[BLOCK_IN__0_:%.+]] = krnl.block [[LOOP_1_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__0_]]) with ([[LOOP_1_]] -> [[I_2_:%.+]] = 0 to 512){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__0_]]) : (!krnl.loop) -> index
+// CHECK-DAG:           [[VAR_50_1_:%.+]] = vector.load [[VAR_reshape_]]{{.}}[[LOAD_PARAM_0_MEM_1_]]{{.}} : memref<512xi32>, vector<32xi32>
+// CHECK-DAG:           [[VAR_51_1_:%.+]] = krnl.load [[RES_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:               [[LOAD_VAR_1_MEM_1_:%.+]] = vector.splat [[VAR_51_1_]] : vector<32xi32>
+// CHECK:               [[VAR_53_1_:%.+]] = arith.subi [[VAR_50_1_]], [[LOAD_VAR_1_MEM_1_]] : vector<32xi32>
+// CHECK:               vector.store [[VAR_53_1_]], [[VAR_reshape_7_]]{{.}}[[LOAD_PARAM_0_MEM_1_]]{{.}} : memref<512xi32>, vector<32xi32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[VAR_14_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_8_:%.+]] = memref.alloc() {{.*}}: memref<32x64xi32>
+// CHECK-DAG:       [[LOOP_2_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_2_]]#0, [[LOOP_2_]]#1) with ([[LOOP_2_]]#0 -> [[I_3_:%.+]] = 0 to 32, [[LOOP_2_]]#1 -> [[I_4_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_48_1_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_2_]]#0, [[LOOP_2_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_:%.+]] = krnl.load [[PARAM_3_]]{{.}}[[VAR_48_1_]]#0, [[VAR_48_1_]]#1] : memref<32x64xui8>
+// CHECK:             [[VAR_50_2_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_0_MEM_1_]] : ui8 to i8
+// CHECK-DAG:         [[VAR_51_2_:%.+]] = arith.extui [[VAR_50_2_]] : i8 to i16
+// CHECK-DAG:         [[LOAD_VAR_1_MEM_1_:%.+]] = krnl.load [[VAR_14_]][] : memref<i16>
+// CHECK:             [[VAR_53_2_:%.+]] = arith.subi [[VAR_51_2_]], [[LOAD_VAR_1_MEM_1_]] : i16
+// CHECK:             [[VAR_54_1_:%.+]] = arith.trunci [[VAR_53_2_]] : i16 to i8
+// CHECK:             [[VAR_55_1_:%.+]] = arith.extsi [[VAR_54_1_]] : i8 to i32
+// CHECK:             krnl.store [[VAR_55_1_]], [[RES_8_]]{{.}}[[VAR_48_1_]]#0, [[VAR_48_1_]]#1] : memref<32x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[VAR_16_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_9_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_PARAM_5_MEM_:%.+]] = krnl.load [[PARAM_5_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xui8>
+// CHECK:           [[VAR_18_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_5_MEM_]] : ui8 to i8
+// CHECK:           [[VAR_19_:%.+]] = arith.extui [[VAR_18_]] : i8 to i16
+// CHECK:           krnl.store [[VAR_19_]], [[RES_9_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_10_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_RES_9_MEM_:%.+]] = krnl.load [[RES_9_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[LOAD_VAR_16_MEM_:%.+]] = krnl.load [[VAR_16_]][] : memref<i16>
+// CHECK:           [[VAR_22_:%.+]] = arith.subi [[LOAD_RES_9_MEM_]], [[LOAD_VAR_16_MEM_]] : i16
+// CHECK:           krnl.store [[VAR_22_]], [[RES_10_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_11_:%.+]] = memref.alloc() {{.*}}: memref<1xi8>
+// CHECK-DAG:       [[LOAD_RES_10_MEM_:%.+]] = krnl.load [[RES_10_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK:           [[VAR_24_:%.+]] = arith.trunci [[LOAD_RES_10_MEM_]] : i16 to i8
+// CHECK:           krnl.store [[VAR_24_]], [[RES_11_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK-DAG:       [[RES_12_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_RES_11_MEM_:%.+]] = krnl.load [[RES_11_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_26_:%.+]] = arith.extsi [[LOAD_RES_11_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_26_]], [[RES_12_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK-DAG:       [[RES_13_:%.+]] = memref.alloc() {{.*}}: memref<32x64xi32>
+// CHECK-DAG:       [[RES_14_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_2048_]], [[RES_14_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_15_:%.+]] = memref.reshape [[RES_8_]]([[RES_14_]]) : (memref<32x64xi32>, memref<1xindex>) -> memref<2048xi32>
+// CHECK-DAG:       [[RES_15_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_2048_]], [[RES_15_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_17_:%.+]] = memref.reshape [[RES_13_]]([[RES_15_]]) : (memref<32x64xi32>, memref<1xindex>) -> memref<2048xi32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_3_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__1_:%.+]], [[BLOCK_IN__1_:%.+]] = krnl.block [[LOOP_3_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__1_]]) with ([[LOOP_3_]] -> [[I_5_:%.+]] = 0 to 2048){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__1_]]) : (!krnl.loop) -> index
+// CHECK-DAG:           [[VAR_50_2_:%.+]] = vector.load [[VAR_reshape_15_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_]]{{.}} : memref<2048xi32>, vector<32xi32>
+// CHECK-DAG:           [[VAR_51_2_:%.+]] = krnl.load [[RES_12_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:               [[LOAD_VAR_1_MEM_1_1_:%.+]] = vector.splat [[VAR_51_2_]] : vector<32xi32>
+// CHECK:               [[VAR_53_3_:%.+]] = arith.subi [[VAR_50_2_]], [[LOAD_VAR_1_MEM_1_1_]] : vector<32xi32>
+// CHECK:               vector.store [[VAR_53_3_]], [[VAR_reshape_17_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_]]{{.}} : memref<2048xi32>, vector<32xi32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[VAR_27_:%.+]] = "krnl.global"() {name = "constant_{{[0-9]+}}", shape = [], value = dense<128> : tensor<i16>} : () -> memref<i16>
+// CHECK-DAG:       [[RES_16_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_PARAM_7_MEM_:%.+]] = krnl.load [[PARAM_7_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xui8>
+// CHECK:           [[VAR_29_:%.+]] = builtin.unrealized_conversion_cast [[LOAD_PARAM_7_MEM_]] : ui8 to i8
+// CHECK:           [[VAR_30_:%.+]] = arith.extui [[VAR_29_]] : i8 to i16
+// CHECK:           krnl.store [[VAR_30_]], [[RES_16_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_17_:%.+]] = memref.alloc() {{.*}}: memref<1xi16>
+// CHECK-DAG:       [[LOAD_RES_16_MEM_:%.+]] = krnl.load [[RES_16_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[LOAD_VAR_27_MEM_:%.+]] = krnl.load [[VAR_27_]][] : memref<i16>
+// CHECK:           [[VAR_33_:%.+]] = arith.subi [[LOAD_RES_16_MEM_]], [[LOAD_VAR_27_MEM_]] : i16
+// CHECK:           krnl.store [[VAR_33_]], [[RES_17_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK-DAG:       [[RES_18_:%.+]] = memref.alloc() {{.*}}: memref<1xi8>
+// CHECK-DAG:       [[LOAD_RES_17_MEM_:%.+]] = krnl.load [[RES_17_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi16>
+// CHECK:           [[VAR_35_:%.+]] = arith.trunci [[LOAD_RES_17_MEM_]] : i16 to i8
+// CHECK:           krnl.store [[VAR_35_]], [[RES_18_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK-DAG:       [[RES_19_:%.+]] = memref.alloc() {{.*}}: memref<1xi32>
+// CHECK-DAG:       [[LOAD_RES_18_MEM_:%.+]] = krnl.load [[RES_18_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi8>
+// CHECK:           [[VAR_37_:%.+]] = arith.extsi [[LOAD_RES_18_MEM_]] : i8 to i32
+// CHECK:           krnl.store [[VAR_37_]], [[RES_19_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:           [[RES_20_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK:           krnl.memset [[RES_20_]], [[CST_0_]] : memref<16x64xi32>
+// CHECK:           [[LOOP_4_:%.+]]:3 = krnl.define_loops 3
+// CHECK:           [[BLOCK_TILE__2_:%.+]], [[BLOCK_IN__2_:%.+]] = krnl.block [[LOOP_4_]]#0 4 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:           [[BLOCK_TILE__3_:%.+]], [[BLOCK_IN__3_:%.+]] = krnl.block [[LOOP_4_]]#1 8 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:           [[BLOCK_TILE__4_:%.+]], [[BLOCK_IN__4_:%.+]] = krnl.block [[LOOP_4_]]#2 8 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:           krnl.permute([[BLOCK_TILE__2_]], [[BLOCK_IN__2_]], [[BLOCK_TILE__2_]]_23, [[BLOCK_IN__2_]]_24, [[BLOCK_TILE__2_]]_25, [[BLOCK_IN__2_]]_26) [0, 3, 1, 4, 2, 5] : !krnl.loop, !krnl.loop, !krnl.loop, !krnl.loop, !krnl.loop, !krnl.loop
+// CHECK:           krnl.iterate([[BLOCK_TILE__2_]], [[BLOCK_TILE__2_]]_23, [[BLOCK_TILE__2_]]_25) with ([[LOOP_4_]]#0 -> [[I_6_:%.+]] = [[CST_0_1_]] to [[CST_16_]], [[LOOP_4_]]#1 -> [[I_7_:%.+]] = [[CST_0_1_]] to [[CST_64_]], [[LOOP_4_]]#2 -> [[I_8_:%.+]] = [[CST_0_1_]] to [[CST_32_]]){
+// CHECK:             [[VAR_48_2_:%.+]]:3 = krnl.get_induction_var_value([[BLOCK_TILE__2_]], [[BLOCK_TILE__2_]]_23, [[BLOCK_TILE__2_]]_25) : (!krnl.loop, !krnl.loop, !krnl.loop) -> (index, index, index)
+// CHECK:             krnl.matmul [[RES_5_]]{{.}}[[CST_0_1_]], [[CST_0_1_]]{{.}}, [[RES_13_]]{{.}}[[CST_0_1_]], [[CST_0_1_]]{{.}}, [[RES_20_]]{{.}}[[CST_0_1_]], [[CST_0_1_]]{{.}}, ([[BLOCK_IN__2_]], [[BLOCK_IN__2_]]_24, [[BLOCK_IN__2_]]_26), ([[VAR_48_2_]]#0, [[VAR_48_2_]]#1, [[VAR_48_2_]]#2), ([[CST_16_]], [[CST_64_]], [[CST_32_]]) {aTileSize = [], bTileSize = [], cTileSize = [], computeTileSize = [4, 8, 8]} : memref<16x32xi32>, memref<32x64xi32>, memref<16x64xi32>, (!krnl.loop, !krnl.loop, !krnl.loop)
+// CHECK:           }
+// CHECK-DAG:       [[RES_21_:%.+]] = memref.alloc() {{.*}}: memref<16x64xf32>
+// CHECK-DAG:       [[LOOP_5_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_5_]]#0, [[LOOP_5_]]#1) with ([[LOOP_5_]]#0 -> [[I_9_:%.+]] = 0 to 16, [[LOOP_5_]]#1 -> [[I_10_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_48_3_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_5_]]#0, [[LOOP_5_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_1_:%.+]] = krnl.load [[RES_20_]]{{.}}[[VAR_48_3_]]#0, [[VAR_48_3_]]#1] : memref<16x64xi32>
+// CHECK:             [[VAR_50_3_:%.+]] = arith.sitofp [[LOAD_PARAM_0_MEM_1_1_]] : i32 to f32
+// CHECK:             krnl.store [[VAR_50_3_]], [[RES_21_]]{{.}}[[VAR_48_3_]]#0, [[VAR_48_3_]]#1] : memref<16x64xf32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_22_:%.+]] = memref.alloc() {{.*}}: memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_1_MEM_:%.+]] = krnl.load [[PARAM_1_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_4_MEM_:%.+]] = krnl.load [[PARAM_4_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:           [[VAR_42_:%.+]] = arith.mulf [[LOAD_PARAM_1_MEM_]], [[LOAD_PARAM_4_MEM_]] : f32
+// CHECK:           krnl.store [[VAR_42_]], [[RES_22_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[RES_23_:%.+]] = memref.alloc() {{.*}}: memref<1xf32>
+// CHECK-DAG:       [[LOAD_RES_22_MEM_:%.+]] = krnl.load [[RES_22_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[LOAD_PARAM_6_MEM_:%.+]] = krnl.load [[PARAM_6_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:           [[VAR_45_:%.+]] = arith.divf [[LOAD_RES_22_MEM_]], [[LOAD_PARAM_6_MEM_]] : f32
+// CHECK:           krnl.store [[VAR_45_]], [[RES_23_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK-DAG:       [[RES_24_:%.+]] = memref.alloc() {{.*}}: memref<16x64xf32>
+// CHECK-DAG:       [[RES_25_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_25_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_32_:%.+]] = memref.reshape [[RES_21_]]([[RES_25_]]) : (memref<16x64xf32>, memref<1xindex>) -> memref<1024xf32>
+// CHECK-DAG:       [[RES_26_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_26_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_34_:%.+]] = memref.reshape [[RES_24_]]([[RES_26_]]) : (memref<16x64xf32>, memref<1xindex>) -> memref<1024xf32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_6_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__5_:%.+]], [[BLOCK_IN__5_:%.+]] = krnl.block [[LOOP_6_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__5_]]) with ([[LOOP_6_]] -> [[I_11_:%.+]] = 0 to 1024){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_1_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__5_]]) : (!krnl.loop) -> index
+// CHECK-DAG:           [[VAR_50_3_:%.+]] = vector.load [[VAR_reshape_32_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_]]{{.}} : memref<1024xf32>, vector<32xf32>
+// CHECK-DAG:           [[VAR_51_2_1_:%.+]] = krnl.load [[RES_23_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xf32>
+// CHECK:               [[LOAD_VAR_1_MEM_1_1_:%.+]] = vector.splat [[VAR_51_2_1_]] : vector<32xf32>
+// CHECK:               [[VAR_53_4_:%.+]] = arith.mulf [[VAR_50_3_]], [[LOAD_VAR_1_MEM_1_1_]] : vector<32xf32>
+// CHECK:               vector.store [[VAR_53_4_]], [[VAR_reshape_34_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_]]{{.}} : memref<1024xf32>, vector<32xf32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[RES_27_:%.+]] = memref.alloc() {{.*}}: memref<16x64xf32>
+// CHECK-DAG:       [[RES_28_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_28_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_37_:%.+]] = memref.reshape [[RES_24_]]([[RES_28_]]) : (memref<16x64xf32>, memref<1xindex>) -> memref<1024xf32>
+// CHECK-DAG:       [[RES_29_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_29_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_39_:%.+]] = memref.reshape [[RES_27_]]([[RES_29_]]) : (memref<16x64xf32>, memref<1xindex>) -> memref<1024xf32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_7_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__6_:%.+]], [[BLOCK_IN__6_:%.+]] = krnl.block [[LOOP_7_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__6_]]) with ([[LOOP_7_]] -> [[I_12_:%.+]] = 0 to 1024){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_1_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__6_]]) : (!krnl.loop) -> index
+// CHECK:               [[VAR_50_3_1_:%.+]] = vector.load [[VAR_reshape_37_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_]]{{.}} : memref<1024xf32>, vector<32xf32>
+// CHECK:               [[VAR_51_3_:%.+]] = vector.shape_cast [[VAR_50_3_1_]] : vector<32xf32> to vector<8x4xf32>
+// CHECK:               [[LOAD_VAR_1_MEM_1_1_1_:%.+]] = vector.extract [[VAR_51_3_]][0] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_53_5_:%.+]] = "krnl.round_even"([[LOAD_VAR_1_MEM_1_1_1_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_54_2_:%.+]] = vector.insert [[VAR_53_5_]], [[VAR_51_3_]] [0] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_55_2_:%.+]] = vector.extract [[VAR_51_3_]][1] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_56_:%.+]] = "krnl.round_even"([[VAR_55_2_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_57_:%.+]] = vector.insert [[VAR_56_]], [[VAR_54_2_]] [1] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_58_:%.+]] = vector.extract [[VAR_51_3_]][2] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_59_:%.+]] = "krnl.round_even"([[VAR_58_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_60_:%.+]] = vector.insert [[VAR_59_]], [[VAR_57_]] [2] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_61_:%.+]] = vector.extract [[VAR_51_3_]][3] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_62_:%.+]] = "krnl.round_even"([[VAR_61_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_63_:%.+]] = vector.insert [[VAR_62_]], [[VAR_60_]] [3] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_64_:%.+]] = vector.extract [[VAR_51_3_]][4] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_65_:%.+]] = "krnl.round_even"([[VAR_64_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_66_:%.+]] = vector.insert [[VAR_65_]], [[VAR_63_]] [4] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_67_:%.+]] = vector.extract [[VAR_51_3_]][5] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_68_:%.+]] = "krnl.round_even"([[VAR_67_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_69_:%.+]] = vector.insert [[VAR_68_]], [[VAR_66_]] [5] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_70_:%.+]] = vector.extract [[VAR_51_3_]][6] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_71_:%.+]] = "krnl.round_even"([[VAR_70_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK-DAG:           [[VAR_72_:%.+]] = vector.insert [[VAR_71_]], [[VAR_69_]] [6] : vector<4xf32> into vector<8x4xf32>
+// CHECK-DAG:           [[VAR_73_:%.+]] = vector.extract [[VAR_51_3_]][7] : vector<4xf32> from vector<8x4xf32>
+// CHECK:               [[VAR_74_:%.+]] = "krnl.round_even"([[VAR_73_]]) : (vector<4xf32>) -> vector<4xf32>
+// CHECK:               [[VAR_75_:%.+]] = vector.insert [[VAR_74_]], [[VAR_72_]] [7] : vector<4xf32> into vector<8x4xf32>
+// CHECK:               [[VAR_76_:%.+]] = vector.shape_cast [[VAR_75_]] : vector<8x4xf32> to vector<32xf32>
+// CHECK:               vector.store [[VAR_76_]], [[VAR_reshape_39_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_]]{{.}} : memref<1024xf32>, vector<32xf32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[RES_30_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK-DAG:       [[LOOP_8_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_8_]]#0, [[LOOP_8_]]#1) with ([[LOOP_8_]]#0 -> [[I_13_:%.+]] = 0 to 16, [[LOOP_8_]]#1 -> [[I_14_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_48_4_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_8_]]#0, [[LOOP_8_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_1_1_1_:%.+]] = krnl.load [[RES_27_]]{{.}}[[VAR_48_4_]]#0, [[VAR_48_4_]]#1] : memref<16x64xf32>
+// CHECK:             [[VAR_50_4_:%.+]] = arith.fptosi [[LOAD_PARAM_0_MEM_1_1_1_1_]] : f32 to i32
+// CHECK:             krnl.store [[VAR_50_4_]], [[RES_30_]]{{.}}[[VAR_48_4_]]#0, [[VAR_48_4_]]#1] : memref<16x64xi32>
+// CHECK:           }
+// CHECK-DAG:       [[RES_31_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK-DAG:       [[RES_32_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_32_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_43_:%.+]] = memref.reshape [[RES_30_]]([[RES_32_]]) : (memref<16x64xi32>, memref<1xindex>) -> memref<1024xi32>
+// CHECK-DAG:       [[RES_33_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_33_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_45_:%.+]] = memref.reshape [[RES_31_]]([[RES_33_]]) : (memref<16x64xi32>, memref<1xindex>) -> memref<1024xi32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_9_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__7_:%.+]], [[BLOCK_IN__7_:%.+]] = krnl.block [[LOOP_9_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__7_]]) with ([[LOOP_9_]] -> [[I_15_:%.+]] = 0 to 1024){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_1_1_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__7_]]) : (!krnl.loop) -> index
+// CHECK-DAG:           [[VAR_50_4_:%.+]] = vector.load [[VAR_reshape_43_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_1_]]{{.}} : memref<1024xi32>, vector<32xi32>
+// CHECK-DAG:           [[VAR_51_3_:%.+]] = krnl.load [[RES_19_]]{{.}}[[CST_0_1_]]{{.}} : memref<1xi32>
+// CHECK:               [[LOAD_VAR_1_MEM_1_1_1_:%.+]] = vector.splat [[VAR_51_3_]] : vector<32xi32>
+// CHECK:               [[VAR_53_6_:%.+]] = arith.addi [[VAR_50_4_]], [[LOAD_VAR_1_MEM_1_1_1_]] : vector<32xi32>
+// CHECK:               vector.store [[VAR_53_6_]], [[VAR_reshape_45_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_1_]]{{.}} : memref<1024xi32>, vector<32xi32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[RES_34_:%.+]] = memref.alloc() {{.*}}: memref<16x64xi32>
+// CHECK-DAG:       [[RES_35_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_35_]][0] : memref<1xindex>
+// CHECK-DAG:       [[VAR_reshape_48_:%.+]] = memref.reshape [[RES_31_]]([[RES_35_]]) : (memref<16x64xi32>, memref<1xindex>) -> memref<1024xi32>
+// CHECK-DAG:       [[RES_36_:%.+]] = memref.alloc() {{.*}}: memref<1xindex>
+// CHECK:           affine.store [[CST_1024_]], [[RES_36_]][0] : memref<1xindex>
+// CHECK:           [[VAR_reshape_50_:%.+]] = memref.reshape [[RES_34_]]([[RES_36_]]) : (memref<16x64xi32>, memref<1xindex>) -> memref<1024xi32>
+// CHECK:           krnl.iterate() with (){
+// CHECK:             [[LOOP_10_:%.+]] = krnl.define_loops 1
+// CHECK:             [[BLOCK_TILE__8_:%.+]], [[BLOCK_IN__8_:%.+]] = krnl.block [[LOOP_10_]] 32 : (!krnl.loop) -> (!krnl.loop, !krnl.loop)
+// CHECK:             krnl.iterate([[BLOCK_TILE__8_]]) with ([[LOOP_10_]] -> [[I_16_:%.+]] = 0 to 1024){
+// CHECK:               [[LOAD_PARAM_0_MEM_1_1_1_1_1_:%.+]] = krnl.get_induction_var_value([[BLOCK_TILE__8_]]) : (!krnl.loop) -> index
+// CHECK-DAG:           [[VAR_50_4_1_:%.+]] = vector.load [[VAR_reshape_48_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_1_1_]]{{.}} : memref<1024xi32>, vector<32xi32>
+// CHECK-DAG:           [[VAR_51_3_1_:%.+]] = krnl.load [[VAR_0_]][] : memref<i32>
+// CHECK:               [[LOAD_VAR_1_MEM_1_1_1_1_:%.+]] = vector.splat [[VAR_51_3_1_]] : vector<32xi32>
+// CHECK:               [[VAR_53_7_:%.+]] = arith.addi [[VAR_50_4_1_]], [[LOAD_VAR_1_MEM_1_1_1_1_]] : vector<32xi32>
+// CHECK:               vector.store [[VAR_53_7_]], [[VAR_reshape_50_]]{{.}}[[LOAD_PARAM_0_MEM_1_1_1_1_1_]]{{.}} : memref<1024xi32>, vector<32xi32>
+// CHECK:             }
+// CHECK:           }
+// CHECK-DAG:       [[RES_37_:%.+]] = memref.alloc() {{.*}}: memref<16x64xui8>
+// CHECK-DAG:       [[LOOP_11_:%.+]]:2 = krnl.define_loops 2
+// CHECK:           krnl.iterate([[LOOP_11_]]#0, [[LOOP_11_]]#1) with ([[LOOP_11_]]#0 -> [[I_17_:%.+]] = 0 to 16, [[LOOP_11_]]#1 -> [[I_18_:%.+]] = 0 to 64){
+// CHECK:             [[VAR_48_5_:%.+]]:2 = krnl.get_induction_var_value([[LOOP_11_]]#0, [[LOOP_11_]]#1) : (!krnl.loop, !krnl.loop) -> (index, index)
+// CHECK:             [[LOAD_PARAM_0_MEM_1_1_1_1_1_:%.+]] = krnl.load [[RES_34_]]{{.}}[[VAR_48_5_]]#0, [[VAR_48_5_]]#1] : memref<16x64xi32>
+// CHECK:             [[VAR_50_5_:%.+]] = arith.trunci [[LOAD_PARAM_0_MEM_1_1_1_1_1_]] : i32 to i8
+// CHECK:             [[VAR_51_4_:%.+]] = builtin.unrealized_conversion_cast [[VAR_50_5_]] : i8 to ui8
+// CHECK:             krnl.store [[VAR_51_4_]], [[RES_37_]]{{.}}[[VAR_48_5_]]#0, [[VAR_48_5_]]#1] : memref<16x64xui8>
+// CHECK:           }
+// CHECK:           return [[RES_37_]] : memref<16x64xui8>
+// CHECK:         }
+}
+


Reviews:
user: login: chentong319
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-14T01:02:01Z
user: login: chentong319
body: To my understanding, the writing lowers the QLinearMatMul to a sequence of onnx op. Should such rewriting be put in decomposition pass?
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-14T01:17:45Z
user: login: tungld
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-14T04:26:31Z
user: login: chentong319
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-14T14:12:49Z
user: login: chentong319
body: LGTM!
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-14T15:57:46Z
user: login: AlexandreEichenberger
body: LGTM, still make "little" sense from a performance's perspective to run quantized models on CPU, compared to better supported float32. Still wonder if a warning would make sense.
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-14T17:45:38Z
user: login: tungld
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-15T04:27:00Z

Review comments:
diff_hunk: @@ -0,0 +1,149 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+//===--------- QLinearMatMul.cpp - Lowering QLinearMatMul Op --------------===//
+//
+// Copyright 2019-2025 The IBM Research Authors.
+//
+// =============================================================================
+//
+// This file lowers the ONNX QLinearMatMul Operator to Krnl dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/Conversion/ONNXToKrnl/ONNXToKrnlCommon.hpp"
+#include "src/Dialect/Krnl/DialectBuilder.hpp"
+#include "src/Dialect/Krnl/KrnlHelper.hpp"
+#include "src/Dialect/Mlir/DialectBuilder.hpp"
+#include "src/Dialect/Mlir/IndexExpr.hpp"
+#include "src/Dialect/ONNX/ONNXOps/ShapeHelper.hpp"
+
+using namespace mlir;
+
+namespace onnx_mlir {
+
+struct ONNXQLinearMatMulOpLowering
+    : public OpConversionPattern<ONNXQLinearMatMulOp> {
+public:
+  ONNXQLinearMatMulOpLowering(TypeConverter &typeConverter, MLIRContext *ctx)
+      : OpConversionPattern(typeConverter, ctx) {}
+
+  LogicalResult matchAndRewrite(ONNXQLinearMatMulOp qlmmOp,
+      ONNXQLinearMatMulOpAdaptor adaptor,
+      ConversionPatternRewriter &rewriter) const final {
+    using LocalDialectBuilder =
+        MultiDialectBuilder<IndexExprBuilderForKrnl, OnnxBuilder>;
+    Operation *op = qlmmOp.getOperation();
+    Location loc = ONNXLoc<ONNXQLinearMatMulOp>(op);
+    LocalDialectBuilder create(rewriter, loc);
+
+    ValueRange operands = adaptor.getOperands();
+    Value A = adaptor.getA();
+    Value aScale = adaptor.getAScale();
+    Value aZeroPoint = adaptor.getAZeroPoint();
+    Value B = adaptor.getB();
+    Value bScale = adaptor.getBScale();
+    Value bZeroPoint = adaptor.getBZeroPoint();
+    Value yScale = adaptor.getYScale();
+    Value yZeroPoint = adaptor.getYZeroPoint();
+
+    // Only support integer8 and float32 now.
+    if (!getElementType(A.getType()).isInteger(8))
path: src/Conversion/ONNXToKrnl/Math/QLinearMatMul.cpp
user: login: chentong319
body: integer8 for A, B and zeroPoint,  and float32 for scale?
created_at: 2025-04-14T01:02:00Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,149 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+//===--------- QLinearMatMul.cpp - Lowering QLinearMatMul Op --------------===//
+//
+// Copyright 2019-2025 The IBM Research Authors.
+//
+// =============================================================================
+//
+// This file lowers the ONNX QLinearMatMul Operator to Krnl dialect.
+//
+//===----------------------------------------------------------------------===//
+
+#include "src/Conversion/ONNXToKrnl/ONNXToKrnlCommon.hpp"
+#include "src/Dialect/Krnl/DialectBuilder.hpp"
+#include "src/Dialect/Krnl/KrnlHelper.hpp"
+#include "src/Dialect/Mlir/DialectBuilder.hpp"
+#include "src/Dialect/Mlir/IndexExpr.hpp"
+#include "src/Dialect/ONNX/ONNXOps/ShapeHelper.hpp"
+
+using namespace mlir;
+
+namespace onnx_mlir {
+
+struct ONNXQLinearMatMulOpLowering
+    : public OpConversionPattern<ONNXQLinearMatMulOp> {
+public:
+  ONNXQLinearMatMulOpLowering(TypeConverter &typeConverter, MLIRContext *ctx)
+      : OpConversionPattern(typeConverter, ctx) {}
+
+  LogicalResult matchAndRewrite(ONNXQLinearMatMulOp qlmmOp,
+      ONNXQLinearMatMulOpAdaptor adaptor,
+      ConversionPatternRewriter &rewriter) const final {
+    using LocalDialectBuilder =
+        MultiDialectBuilder<IndexExprBuilderForKrnl, OnnxBuilder>;
+    Operation *op = qlmmOp.getOperation();
+    Location loc = ONNXLoc<ONNXQLinearMatMulOp>(op);
+    LocalDialectBuilder create(rewriter, loc);
+
+    ValueRange operands = adaptor.getOperands();
+    Value A = adaptor.getA();
+    Value aScale = adaptor.getAScale();
+    Value aZeroPoint = adaptor.getAZeroPoint();
+    Value B = adaptor.getB();
+    Value bScale = adaptor.getBScale();
+    Value bZeroPoint = adaptor.getBZeroPoint();
+    Value yScale = adaptor.getYScale();
+    Value yZeroPoint = adaptor.getYZeroPoint();
+
+    // Only support integer8 and float32 now.
+    if (!getElementType(A.getType()).isInteger(8))
path: src/Conversion/ONNXToKrnl/Math/QLinearMatMul.cpp
user: login: tungld
body: Yes, made it clear now. Thanks!
created_at: 2025-04-14T04:26:31Z
author_association: MEMBER
diff_hunk: @@ -891,4 +891,27 @@ bool isElementAttrUninitializedDenseResource(ElementsAttr elementsAttr) {
          !denseResourceElementsAttr.getRawHandle().getBlob();
 }
 
+//===----------------------------------------------------------------------===//
+// Helper for quantization
+//===----------------------------------------------------------------------===//
+Value getOrCastToI8(
path: src/Dialect/ONNX/ONNXOps/OpHelper.cpp
user: login: chentong319
body: Should this function be DialectBuilder?
created_at: 2025-04-14T14:12:49Z
author_association: COLLABORATOR
diff_hunk: @@ -891,4 +891,27 @@ bool isElementAttrUninitializedDenseResource(ElementsAttr elementsAttr) {
          !denseResourceElementsAttr.getRawHandle().getBlob();
 }
 
+//===----------------------------------------------------------------------===//
+// Helper for quantization
+//===----------------------------------------------------------------------===//
+Value getOrCastToI8(
path: src/Dialect/ONNX/ONNXOps/OpHelper.cpp
user: login: tungld
body: Yes, done. Thanks!
created_at: 2025-04-15T04:27:00Z
author_association: MEMBER

PR comments:
user: login: tungld
created_at: 2025-04-14T04:22:24Z
body: > To my understanding, the writing lowers the QLinearMatMul to a sequence of onnx op. Should such rewriting be put in decomposition pass?

It can be in the decomposition pass. However, we want to keep it as it is in case of NNPA (or other accelerators). So we need a way (it may be similar to turning off constant prop patterns) for turning off the decomposition when compiling for NNPA, which is currently not available. So that's why I decompose it during onnx-to-krnl.
author_association: MEMBER
user: login: chentong319
created_at: 2025-04-14T13:48:22Z
body: In general, onnx-mlir should provide a fine control on graph level optimization for different target architecture/accelerator. We can discuss such mechanism later.
author_association: COLLABORATOR
user: login: tungld
created_at: 2025-04-15T07:12:18Z
body: @jenkins-droid test this please
author_association: MEMBER
user: login: jenkins-droid
created_at: 2025-04-16T04:48:42Z
body: **Jenkins Linux amd64** Build [#16504](https://www.onnxmlir.xyz/jenkinx/job/ONNX-MLIR-Pipeline-Docker-Build/16504/) [push] null... failed after 1 hr 23 min
author_association: COLLABORATOR
user: login: jenkins-droid
created_at: 2025-04-16T05:05:56Z
body: **Jenkins Linux ppc64le** Build [#15487](https://www.onnxmlir.xyz/jenkinp/job/ONNX-MLIR-Pipeline-Docker-Build/15487/) [push] null... aborted after 1 hr 40 min
author_association: COLLABORATOR
user: login: jenkins-droid
created_at: 2025-04-16T05:05:59Z
body: **Jenkins Linux s390x** Build [#16506](https://www.onnxmlir.xyz/jenkins/job/ONNX-MLIR-Pipeline-Docker-Build/16506/) [push] null... aborted after 1 hr 40 min
author_association: COLLABORATOR