Repository: iree-org/iree

Issue #15760: [CPU] Missing ukernel for f32, i8 -> f32 mmt4d?
Created: 2023-12-01T14:16:45Z
This dispatch shows up after all the recent DT fixes. When we enable ukernels (all), it doesn't lower to a ukernel but to scalar code. I guess we don't have ukernels for f32, i8 -> f32 mmt4d? Could we add them for x86 and ARM?
It would also be a good idea to fall back to the right codegen path when a ukernel is not available/found.
Thanks!
hal.executable public @main_dispatch_1187 {
  hal.executable.variant public @system_elf_x86_64 target(<"llvm-cpu", "system-elf-x86_64", {cpu = "cascadelake", cpu_features = "+cmov,+mmx,+popcnt,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+avx,+avx2,+fma,+avx512f,+bmi,+bmi2,+aes,+pclmul,+avx512vl,+avx512bw,+avx512dq,+avx512cd,+avx512vnni,+adx,+clflushopt,+clwb,+cx16,+cx8,+crc32,+f16c,+fsgsbase,+fxsr,+invpcid,+lzcnt,+movbe,+pku,+prfchw,+rdrnd,+rdseed,+sahf,+x87,+xsave,+xsavec,+xsaveopt,+xsaves,+evex512", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", link_embedded = false, native_vector_size = 64 : index, target_triple = "x86_64-unknown-linux-gnu", ukernels = "all"}>) {
    hal.executable.export public @main_dispatch_1187_mmt4d_1x600x256x1x16x1_f32xi8xf32 ordinal(0) layout(#hal.pipeline.layout<push_constants = 0, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) {
    ^bb0(%arg0: !hal.device):
      %x, %y, %z = flow.dispatch.workgroup_count_from_slice
      hal.return %x, %y, %z : index, index, index
    }
    builtin.module {
      func.func @main_dispatch_1187_mmt4d_1x600x256x1x16x1_f32xi8xf32() {
        %c128 = arith.constant 128 : index
        %c6272 = arith.constant 6272 : index
        %cst = arith.constant dense<0> : tensor<600x256x16x1xi8>
        %cst_0 = arith.constant 0.000000e+00 : f32
        %0 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c128) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<1x256x1x1xf32>>
        %1 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c6272) : !flow.dispatch.tensor<writeonly:tensor<1x600x1x16xf32>>
        %2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [1, 256, 1, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<1x256x1x1xf32>> -> tensor<1x256x1x1xf32>
        %3 = tensor.empty() : tensor<1x600x1x16xf32>
        %4 = linalg.fill ins(%cst_0 : f32) outs(%3 : tensor<1x600x1x16xf32>) -> tensor<1x600x1x16xf32>
        %5 = linalg.mmt4d ins(%2, %cst : tensor<1x256x1x1xf32>, tensor<600x256x16x1xi8>) outs(%4 : tensor<1x600x1x16xf32>) -> tensor<1x600x1x16xf32>
        flow.dispatch.tensor.store %5, %1, offsets = [0, 0, 0, 0], sizes = [1, 600, 1, 16], strides = [1, 1, 1, 1] : tensor<1x600x1x16xf32> -> !flow.dispatch.tensor<writeonly:tensor<1x600x1x16xf32>>
        return
      }
    }
  }
}

Duplicate issue #15759: [CPU] Missing ukernel for f32, i8 -> f32 mmt4d
Created: 2023-12-01T14:15:39Z
This dispatch shows up after all the recent DT fixes. When we enable ukernels (all), it doesn't lower to a ukernel but to scalar code. I guess we don't have ukernels for f32, i8 -> f32 mmt4d? Could we add them for x86 and ARM?
It would also be a good idea to fall back to the right codegen path when a ukernel is not available/found.
Thanks!
hal.executable public @main_dispatch_1187 {
  hal.executable.variant public @system_elf_x86_64 target(<"llvm-cpu", "system-elf-x86_64", {cpu = "cascadelake", cpu_features = "+cmov,+mmx,+popcnt,+sse,+sse2,+sse3,+ssse3,+sse4.1,+sse4.2,+avx,+avx2,+fma,+avx512f,+bmi,+bmi2,+aes,+pclmul,+avx512vl,+avx512bw,+avx512dq,+avx512cd,+avx512vnni,+adx,+clflushopt,+clwb,+cx16,+cx8,+crc32,+f16c,+fsgsbase,+fxsr,+invpcid,+lzcnt,+movbe,+pku,+prfchw,+rdrnd,+rdseed,+sahf,+x87,+xsave,+xsavec,+xsaveopt,+xsaves,+evex512", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-i128:128-f80:128-n8:16:32:64-S128", link_embedded = false, native_vector_size = 64 : index, target_triple = "x86_64-unknown-linux-gnu", ukernels = "all"}>) {
    hal.executable.export public @main_dispatch_1187_mmt4d_1x600x256x1x16x1_f32xi8xf32 ordinal(0) layout(#hal.pipeline.layout<push_constants = 0, sets = [<0, bindings = [<0, storage_buffer, ReadOnly>, <1, storage_buffer>]>]>) {
    ^bb0(%arg0: !hal.device):
      %x, %y, %z = flow.dispatch.workgroup_count_from_slice
      hal.return %x, %y, %z : index, index, index
    }
    builtin.module {
      func.func @main_dispatch_1187_mmt4d_1x600x256x1x16x1_f32xi8xf32() {
        %c128 = arith.constant 128 : index
        %c6272 = arith.constant 6272 : index
        %cst = arith.constant dense<0> : tensor<600x256x16x1xi8>
        %cst_0 = arith.constant 0.000000e+00 : f32
        %0 = hal.interface.binding.subspan set(0) binding(0) type(storage_buffer) alignment(64) offset(%c128) flags(ReadOnly) : !flow.dispatch.tensor<readonly:tensor<1x256x1x1xf32>>
        %1 = hal.interface.binding.subspan set(0) binding(1) type(storage_buffer) alignment(64) offset(%c6272) : !flow.dispatch.tensor<writeonly:tensor<1x600x1x16xf32>>
        %2 = flow.dispatch.tensor.load %0, offsets = [0, 0, 0, 0], sizes = [1, 256, 1, 1], strides = [1, 1, 1, 1] : !flow.dispatch.tensor<readonly:tensor<1x256x1x1xf32>> -> tensor<1x256x1x1xf32>
        %3 = tensor.empty() : tensor<1x600x1x16xf32>
        %4 = linalg.fill ins(%cst_0 : f32) outs(%3 : tensor<1x600x1x16xf32>) -> tensor<1x600x1x16xf32>
        %5 = linalg.mmt4d ins(%2, %cst : tensor<1x256x1x1xf32>, tensor<600x256x16x1xi8>) outs(%4 : tensor<1x600x1x16xf32>) -> tensor<1x600x1x16xf32>
        flow.dispatch.tensor.store %5, %1, offsets = [0, 0, 0, 0], sizes = [1, 600, 1, 16], strides = [1, 1, 1, 1] : tensor<1x600x1x16xf32> -> !flow.dispatch.tensor<writeonly:tensor<1x600x1x16xf32>>
        return
      }
    }
  }
}

Closing comment (2023-12-01T18:13:21Z): I think this is a dup of #15760, closing the issue