[Codegen][CPU] Lower RISC-V V f16 vfmacc, vfwmacc, and Zvfhmin inner_tiled MMA. (#25000) Lowers the RISC-V V f16 `inner_tiled` MMA paths. Native f16 enums are already on main from #24858. This adds lowering, the widening `vfwmacc.vf` pair, and the `+zvfhmin`-only fallbacks. All of them reuse `lowerRiscvVFmaccLike` and the hoisted accumulator legalization from #24933. Tile N is `vlen/8`. | Encoding | `+zvfh` | `+zvfhmin` only | | --- | --- | --- | | f16×f16→f16 | `llvm.riscv.vfmacc` (`MMA_RISCV_V_VFMACC_*_F16_F16`) | `extf` the accumulator to f32, f32 `vfmacc`, `truncf` back (`*_F16_F16_WIDENF32`) | | f16×f16→f32 | `llvm.riscv.vfwmacc` (`MMA_RISCV_V_VFWMACC_*_F32_F16`) | `extf` both panels, then f32 `vfmacc` (`*_F32_F16_CASTF32`) | `+zvfh` implies `+zvfhmin`. When both are legal, the earlier enum wins: native `vfmacc` for an f16 accumulator, `vfwmacc` for an f32 accumulator. At VLEN=256 the cost model picks `intrinsics_m` 14 (native), 7 (`vfwmacc`), and 6 (both `+zvfhmin` paths). CASTF32 and WIDENF32 bill the widened f32 vector operand as f32. Performance is tracked on #24932. On the X60 (VLEN=256), 1-thread inner_tiled is faster than the mmt4d ukernel for native f16 and `vfwmacc`. The 4-thread and 8-thread large shapes are still slower. Native f16 is non-widening `vfmacc`; the f16 ukernel widens with `vfwmacc`, so that ratio is not the same instruction sequence. `vfwmacc` is the closer comparison. Assisted-by: Cursor Towards #24311 --------- Signed-off-by: Terence <terrencemahlatini@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com>
diff --git a/compiler/src/iree/compiler/Codegen/Common/test/materialize_encoding_riscv_inner_tiled.mlir b/compiler/src/iree/compiler/Codegen/Common/test/materialize_encoding_riscv_inner_tiled.mlir index e5ade49..ff14c0b 100644 --- a/compiler/src/iree/compiler/Codegen/Common/test/materialize_encoding_riscv_inner_tiled.mlir +++ b/compiler/src/iree/compiler/Codegen/Common/test/materialize_encoding_riscv_inner_tiled.mlir
@@ -281,3 +281,190 @@ // CHECK-LABEL: func @matmul_f32_plain_v( // CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled // CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, intrinsics_m = 6, vlen = 128> + +// ----- + +// f16→f32 CASTF32 via Zvfhmin. N tile is vlen/8. The widened RHS is LMUL=4, +// so the register budget picks intrinsics_m = 6. +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +func.func @matmul_f16_castf32_zvl128b(%arg0 : tensor<?x?xf16>, %arg1 : tensor<?x?xf16>, %m: index, %n: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvfhmin,+zvl128b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf16> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf16> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf16, #lhs>, tensor<?x?xf16, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %n, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f16_castf32_zvl128b( +// CHECK: %[[PACK_LHS:.+]] = linalg.pack {{.*}}inner_tiles = [6, 1] +// CHECK-SAME: -> tensor<?x?x6x1xf16> +// CHECK: %[[PACK_RHS:.+]] = linalg.pack {{.*}}inner_tiles = [16, 1] +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32, intrinsics_m = 6, vlen = 128> + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +func.func @matmul_f16_castf32_zvl256b(%arg0 : tensor<?x?xf16>, %arg1 : tensor<?x?xf16>, %m: index, %n: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvfhmin,+zvl256b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf16> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf16> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf16, #lhs>, tensor<?x?xf16, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %n, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f16_castf32_zvl256b( +// CHECK: %[[PACK_RHS:.+]] = linalg.pack {{.*}}inner_tiles = [32, 1] +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32, intrinsics_m = 6, vlen = 256> + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +func.func @matmul_f16_castf32_zvl512b(%arg0 : tensor<?x?xf16>, %arg1 : tensor<?x?xf16>, %m: index, %n: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvfhmin,+zvl512b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf16> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf16> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf16, #lhs>, tensor<?x?xf16, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %n, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f16_castf32_zvl512b( +// CHECK: %[[PACK_RHS:.+]] = linalg.pack {{.*}}inner_tiles = [64, 1] +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32, intrinsics_m = 6, vlen = 512> + +// ----- + +// +zvfh f16×f16→f32 is vfwmacc.vf. RHS stays f16, so intrinsics_m = 7. +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +func.func @matmul_f16_vfwmacc_zvfh(%arg0 : tensor<?x?xf16>, %arg1 : tensor<?x?xf16>, %m: index, %n: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvfh,+zvl256b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf16> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf16> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf16, #lhs>, tensor<?x?xf16, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %n, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f16_vfwmacc_zvfh( +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16, intrinsics_m = 7, vlen = 256> + +// ----- + +// Narrow-N CASTF32. The swapped tile's vector operand is the LHS, billed at +// f32 width, so intrinsics_n = 6. +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, 8, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, 8, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f16, f16, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, 8, ?]> +func.func @matmul_f16_castf32_zvl256b_narrow_n(%arg0 : tensor<?x?xf16>, %arg1 : tensor<?x?xf16>, %m: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvfhmin,+zvl256b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf16> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf16> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf16, #lhs>, tensor<?x?xf16, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f16_castf32_zvl256b_narrow_n( +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32, intrinsics_n = 6, vlen = 256> + +// ----- + +// Zvfhmin f16×f16→f16 widens the accumulator to f32. intrinsics_m = 6. +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f16, f16, f16], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f16, f16, f16], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f16, f16, f16], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +func.func @matmul_f16_widenf32_zvfhmin_zvl256b(%arg0 : tensor<?x?xf16>, %arg1 : tensor<?x?xf16>, %m: index, %n: index, %k: index) -> tensor<?x?xf16> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvfhmin,+zvl256b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f16 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf16> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf16> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %n, %k} : tensor<?x?xf16> -> tensor<?x?xf16, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf16, #acc> + %3 = linalg.fill ins(%cst : f16) outs(%2 : tensor<?x?xf16, #acc>) -> tensor<?x?xf16, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf16, #lhs>, tensor<?x?xf16, #rhs>) + outs(%3 : tensor<?x?xf16, #acc>) -> tensor<?x?xf16, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %n, %k} : tensor<?x?xf16, #acc> -> tensor<?x?xf16>{%d0, %d1} + return %5 : tensor<?x?xf16> +} +// CHECK-LABEL: func @matmul_f16_widenf32_zvfhmin_zvl256b( +// CHECK: %[[PACK_LHS:.+]] = linalg.pack {{.*}}inner_tiles = [6, 1] +// CHECK: %[[PACK_RHS:.+]] = linalg.pack {{.*}}inner_tiles = [32, 1] +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32, intrinsics_m = 6, vlen = 256>
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUAttrs.cpp b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUAttrs.cpp index 2d13e54..71f77f3 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUAttrs.cpp +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUAttrs.cpp
@@ -490,7 +490,13 @@ case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32: case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32: case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16: - case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16: { + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16: + case MMAIntrinsic::MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16: + case MMAIntrinsic::MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32: { int64_t vl = vlen / 8; return isMNSwapped(intrinsic) ? Tuple{vl, 1, 1} : Tuple{1, vl, 1}; } @@ -748,11 +754,17 @@ return {f32, f32, f32}; case MMAIntrinsic::MMA_X86_AVX512_1x16x1_F32_F16_CASTF32: case MMAIntrinsic::MMA_X86_AVX512_16x1x1_F32_F16_CASTF32: + case MMAIntrinsic::MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16: + case MMAIntrinsic::MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32: return {f16, f16, f32}; case MMAIntrinsic::MMA_X86_AVX512FP16_1x32x1_F16_F16: case MMAIntrinsic::MMA_X86_AVX512FP16_32x1x1_F16_F16: case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16: case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32: return {f16, f16, f16}; case MMAIntrinsic::MMA_X86_AVX512BF16_1x16x2_F32_BF16: case MMAIntrinsic::MMA_X86_AVX512BF16_16x1x2_F32_BF16: @@ -1001,7 +1013,7 @@ return vector::ScalableExtractOp::create(b, loc, fixedTy, v, /*pos=*/0); } -// RVV `.vf` FMA (`vfmacc` / `vfwmaccbf16`). Enum bit 0 selects which operand +// RVV `.vf` FMA (`vfmacc` / `vfwmacc`). Enum bit 0 selects which operand // is the scalar (swapped => RHS). The vector operand is inserted into nxv8 so // 8 * vscale == vlen/8 lanes. ACC is left scalable when the caller already // legalized it (hoisted out of K). frm=7 is DYN; policy=1 is tail-agnostic. @@ -1064,10 +1076,17 @@ return lowerX86Avx512Vnni16x16x2I8(builder, loc, lhs, rhs, acc); } if (intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32 || - intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32) { + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16) { return lowerRiscvVFmaccLike(builder, loc, intrinsic, vlen, lhs, rhs, acc, "llvm.riscv.vfmacc"); } + if (intrinsic == MMAIntrinsic::MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16) { + return lowerRiscvVFmaccLike(builder, loc, intrinsic, vlen, lhs, rhs, acc, + "llvm.riscv.vfwmacc"); + } // Sign-/float-extend a vector to a wider element type. Used by the // *_CASTF32 (f16 → f32) and *_CASTI16 (i8 → i16) variants where the // intrinsic only exists at the wider type. @@ -1080,6 +1099,16 @@ } return arith::ExtSIOp::create(builder, loc, wideTy, v); }; + if (intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32) { + Type f32 = builder.getF32Type(); + Value wideLhs = widen(lhs, f32); + Value wideRhs = widen(rhs, f32); + return lowerRiscvVFmaccLike(builder, loc, intrinsic, vlen, wideLhs, wideRhs, + acc, "llvm.riscv.vfmacc"); + } // For *_CAST* intrinsics, widen lhs/rhs to the intrinsic's element type // *before* the broadcast below. The alternative — widening after the @@ -1354,13 +1383,42 @@ getVlen()); }; if (intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32 || - intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32) { + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32) { int64_t lanes = getVlen() / 8; - auto legalizeAcc = [](OpBuilder &b, Location loc, Value v) -> Value { + bool widenF16Acc = + intrinsic == + MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32; + auto legalizeAcc = [widenF16Acc](OpBuilder &b, Location loc, + Value v) -> Value { + // Zvfhmin has no f16 MAC. This pair is lossy (extf(truncf(x)) != x): + // hoisting keeps the acc in f32 across K and skips per-K f16 rounding. + // TODO(#25058): honor --iree-llvmcpu-skip-intermediate-roundings=false. + if (widenF16Acc) { + Type f32 = b.getF32Type(); + auto vt = cast<VectorType>(v.getType()); + v = arith::ExtFOp::create(b, loc, VectorType::get(vt.getShape(), f32), + v, arith::FastMathFlagsAttr{}); + } return insertFixedIntoNxv8(b, loc, v); }; - auto restoreAcc = [lanes](OpBuilder &b, Location loc, Value v) -> Value { - return extractNxv8ToFixed(b, loc, v, lanes); + auto restoreAcc = [lanes, widenF16Acc](OpBuilder &b, Location loc, + Value v) -> Value { + v = extractNxv8ToFixed(b, loc, v, lanes); + if (widenF16Acc) { + auto vt = cast<VectorType>(v.getType()); + v = arith::TruncFOp::create( + b, loc, VectorType::get(vt.getShape(), b.getF16Type()), v); + } + return v; }; return Codegen::buildDataTiledMMAUnderlyingOperations( builder, loc, getSwizzle(*this, /*operandIdx=*/0),
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUEnums.td b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUEnums.td index c730ae1..7d43ba3 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUEnums.td +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUEnums.td
@@ -191,6 +191,28 @@ def MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16 : I32EnumAttrCase<"MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16", 0x3121>; +// RISC-V V `vfwmacc.vf` (f16×f16→f32). Lane count is vlen/8. The vector +// operand stays f16 (LMUL=2) and the accumulator is f32 (LMUL=4). +// Requires +zvfh. +def MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16 + : I32EnumAttrCase<"MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16", 0x3122>; +def MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16 + : I32EnumAttrCase<"MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16", 0x3123>; + +// Zvfhmin f16×f16→f32: extf the vector operand, then f32 vfmacc.vf. The +// widened vector is LMUL=4. Requires +zvfhmin. +zvfh uses vfwmacc.vf. +def MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32 + : I32EnumAttrCase<"MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32", 0x3124>; +def MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32 + : I32EnumAttrCase<"MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32", 0x3125>; + +// Zvfhmin f16×f16→f16. The accumulator is widened to f32 for vfmacc.vf and +// narrowed after the MAC. Requires +zvfhmin. +zvfh uses native f16 vfmacc. +def MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32 + : I32EnumAttrCase<"MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32", 0x3126>; +def MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32 + : I32EnumAttrCase<"MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32", 0x3127>; + // Architecture-agnostic, type-polymorphic generic-scalar fallback. Tile // shape is 1×1×1, lowering is a single `vector.contract` over the // unrolled (`intrinsics_m`, `intrinsics_n`, `intrinsics_k`) tile. Unlike @@ -244,6 +266,12 @@ MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32, MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16, MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16, + MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16, + MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16, + MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32, + MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32, + MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32, + MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32, // Architecture-agnostic generic-scalar fallback (type-polymorphic). MMA_GENERIC_SCALAR_1x1x1_REG8, MMA_GENERIC_SCALAR_1x1x1_REG16]>;
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/iree_cpu_inner_tiled_ops.mlir b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/iree_cpu_inner_tiled_ops.mlir index c7d6aba..09cecb8 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/iree_cpu_inner_tiled_ops.mlir +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/iree_cpu_inner_tiled_ops.mlir
@@ -376,3 +376,27 @@ // CHECK: iree_codegen.inner_tiled ins(%arg0, %arg1) outs(%arg2) // CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32, vlen = 128> // CHECK-SAME: semantics = #iree_cpu.mma_semantics<> + +// ----- + +#contraction_accesses = [ + affine_map<(i, j, k) -> (i, k)>, + affine_map<(i, j, k) -> (k, j)>, + affine_map<(i, j, k) -> (i, j)> +] +// MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32 at vlen = 256, intrinsics 1x1x1 +func.func @cpu_riscv_v_vfmacc_f16_castf32( + %lhs: vector<1x1x1xf16>, %rhs: vector<1x1x32xf16>, %acc: vector<1x1x32xf32>) + -> vector<1x1x32xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses, + iterator_types = [#linalg.iterator_type<parallel>, #linalg.iterator_type<parallel>, #linalg.iterator_type<reduction>], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + > : vector<1x1x1xf16>, vector<1x1x32xf16> into vector<1x1x32xf32> + return %0 : vector<1x1x32xf32> +} +// CHECK-LABEL: func @cpu_riscv_v_vfmacc_f16_castf32 +// CHECK: iree_codegen.inner_tiled ins(%arg0, %arg1) outs(%arg2) +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32, vlen = 256> +// CHECK-SAME: semantics = #iree_cpu.mma_semantics<>
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/lower_inner_tiled.mlir b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/lower_inner_tiled.mlir index 6a4109b..7916ff0 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/lower_inner_tiled.mlir +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/lower_inner_tiled.mlir
@@ -565,3 +565,417 @@ // CHECK-NOT: vector.scalable.extract // CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" // CHECK: vector.scalable.extract {{.*}} : vector<32xf32> from vector<[8]xf32> + +// ----- + +// RISC-V VLEN=256 1×32×1 f16: scalar .vf. + +#contraction_accesses_rvv_f16 = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_1x8vlsx1_f16_vlen256( + %lhs: vector<1x1xf16>, %rhs: vector<32x1xf16>, %acc: vector<1x32xf16>) + -> vector<1x32xf16> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses_rvv_f16, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + > : vector<1x1xf16>, vector<32x1xf16> into vector<1x32xf16> + return %0 : vector<1x32xf16> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_1x8vlsx1_f16_vlen256 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: ub.poison : vector<[8]xf16> +// CHECK: util.hoistable_conversion "data_tiled_acc_distribute" +// CHECK: vector.scalable.insert {{.*}} : vector<32xf16> into vector<[8]xf16> +// CHECK: vector.extract {{.*}} : f16 from vector<{{1x1|1}}xf16> +// CHECK-NOT: vector.broadcast {{.*}} : f16 to vector<32xf16> +// CHECK-NOT: vector.broadcast {{.*}} : f16 to vector<[8]xf16> +// CHECK: vector.scalable.insert {{.*}} : vector<32xf16> into vector<[8]xf16> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[POL]]) : (vector<[8]xf16>, f16, vector<[8]xf16>, i64, i64, i64) -> vector<[8]xf16> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" +// CHECK: vector.scalable.extract {{.*}} : vector<32xf16> from vector<[8]xf16> + +// ----- + +// RISC-V VLEN=128 f16: lane count tracks vlen/8. + +#contraction_accesses_rvv_f16_128 = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_1x8vlsx1_f16_vlen128( + %lhs: vector<1x1xf16>, %rhs: vector<16x1xf16>, %acc: vector<1x16xf16>) + -> vector<1x16xf16> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses_rvv_f16_128, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16, vlen = 128>, + semantics = #iree_cpu.mma_semantics<> + > : vector<1x1xf16>, vector<16x1xf16> into vector<1x16xf16> + return %0 : vector<1x16xf16> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_1x8vlsx1_f16_vlen128 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: ub.poison : vector<[8]xf16> +// CHECK: util.hoistable_conversion "data_tiled_acc_distribute" +// CHECK: vector.scalable.insert {{.*}} : vector<16xf16> into vector<[8]xf16> +// CHECK: vector.extract {{.*}} : f16 from vector<{{1x1|1}}xf16> +// CHECK-NOT: vector.broadcast {{.*}} : f16 to vector<[8]xf16> +// CHECK: vector.scalable.insert {{.*}} : vector<16xf16> into vector<[8]xf16> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[POL]]) : (vector<[8]xf16>, f16, vector<[8]xf16>, i64, i64, i64) -> vector<[8]xf16> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" +// CHECK: vector.scalable.extract {{.*}} : vector<16xf16> from vector<[8]xf16> + +// ----- + +// RISC-V VLEN=256 f16→f32 CASTF32. + +#contraction_accesses_rvv_castf32 = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_1x8vlsx1_f16_castf32_vlen256( + %lhs: vector<1x1xf16>, %rhs: vector<32x1xf16>, %acc: vector<1x32xf32>) + -> vector<1x32xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses_rvv_castf32, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + > : vector<1x1xf16>, vector<32x1xf16> into vector<1x32xf32> + return %0 : vector<1x32xf32> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_1x8vlsx1_f16_castf32_vlen256 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: ub.poison : vector<[8]xf32> +// CHECK: util.hoistable_conversion "data_tiled_acc_distribute" +// CHECK: vector.scalable.insert {{.*}} : vector<32xf32> into vector<[8]xf32> +// CHECK: arith.extf {{.*}} : vector<{{1x1|1}}xf16> to vector<{{1x1|1}}xf32> +// CHECK: arith.extf {{.*}} : vector<{{32x1|32}}xf16> to vector<{{32x1|32}}xf32> +// CHECK: vector.extract {{.*}} : f32 from vector<{{1x1|1}}xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<32xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<[8]xf32> +// CHECK: vector.scalable.insert {{.*}} : vector<32xf32> into vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[POL]]) : (vector<[8]xf32>, f32, vector<[8]xf32>, i64, i64, i64) -> vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" +// CHECK: vector.scalable.extract {{.*}} : vector<32xf32> from vector<[8]xf32> + +// ----- + +// RISC-V VLEN=256 swapped CASTF32: scalar from unit-N RHS. + +#contraction_accesses_rvv_castf32_t = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_8vlsx1x1_f16_castf32_vlen256( + %lhs: vector<32x1xf16>, %rhs: vector<1x1xf16>, %acc: vector<32x1xf32>) + -> vector<32x1xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses_rvv_castf32_t, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + > : vector<32x1xf16>, vector<1x1xf16> into vector<32x1xf32> + return %0 : vector<32x1xf32> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_8vlsx1x1_f16_castf32_vlen256 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: ub.poison : vector<[8]xf32> +// CHECK: util.hoistable_conversion "data_tiled_acc_distribute" +// CHECK: vector.scalable.insert {{.*}} : vector<32xf32> into vector<[8]xf32> +// CHECK: arith.extf {{.*}} : vector<{{32x1|32}}xf16> to vector<{{32x1|32}}xf32> +// CHECK: arith.extf {{.*}} : vector<{{1x1|1}}xf16> to vector<{{1x1|1}}xf32> +// CHECK: vector.extract {{.*}} : f32 from vector<{{1x1|1}}xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<32xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<[8]xf32> +// CHECK: vector.scalable.insert {{.*}} : vector<32xf32> into vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[POL]]) : (vector<[8]xf32>, f32, vector<[8]xf32>, i64, i64, i64) -> vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" +// CHECK: vector.scalable.extract {{.*}} : vector<32xf32> from vector<[8]xf32> + +// ----- + +// RISC-V VLEN=128 f16→f32 CASTF32. + +#contraction_accesses_rvv_castf32_128 = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_1x8vlsx1_f16_castf32_vlen128( + %lhs: vector<1x1xf16>, %rhs: vector<16x1xf16>, %acc: vector<1x16xf32>) + -> vector<1x16xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses_rvv_castf32_128, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32, vlen = 128>, + semantics = #iree_cpu.mma_semantics<> + > : vector<1x1xf16>, vector<16x1xf16> into vector<1x16xf32> + return %0 : vector<1x16xf32> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_1x8vlsx1_f16_castf32_vlen128 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: ub.poison : vector<[8]xf32> +// CHECK: util.hoistable_conversion "data_tiled_acc_distribute" +// CHECK: vector.scalable.insert {{.*}} : vector<16xf32> into vector<[8]xf32> +// CHECK: arith.extf {{.*}} : vector<{{1x1|1}}xf16> to vector<{{1x1|1}}xf32> +// CHECK: arith.extf {{.*}} : vector<{{16x1|16}}xf16> to vector<{{16x1|16}}xf32> +// CHECK: vector.extract {{.*}} : f32 from vector<{{1x1|1}}xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<[8]xf32> +// CHECK: vector.scalable.insert {{.*}} : vector<16xf32> into vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[POL]]) : (vector<[8]xf32>, f32, vector<[8]xf32>, i64, i64, i64) -> vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" +// CHECK: vector.scalable.extract {{.*}} : vector<16xf32> from vector<[8]xf32> + +// ----- + +// RISC-V VLEN=256 swapped f16: scalar from unit-N RHS. + +#contraction_accesses_rvv_f16_t = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_8vlsx1x1_f16_vlen256( + %lhs: vector<32x1xf16>, %rhs: vector<1x1xf16>, %acc: vector<32x1xf16>) + -> vector<32x1xf16> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses_rvv_f16_t, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + > : vector<32x1xf16>, vector<1x1xf16> into vector<32x1xf16> + return %0 : vector<32x1xf16> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_8vlsx1x1_f16_vlen256 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: %[[VL:.+]] = arith.constant 32 : i64 +// CHECK: ub.poison : vector<[8]xf16> +// CHECK: vector.extract {{.*}} : f16 from vector<{{1x1|1}}xf16> +// CHECK-NOT: vector.broadcast {{.*}} : f16 to vector<[8]xf16> +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[VL]], %[[POL]]) : (vector<[8]xf16>, f16, vector<[8]xf16>, i64, i64, i64) -> vector<[8]xf16> + +// ----- + +// RISC-V VLEN=256 f16×f16→f32 vfwmacc.vf. No extf; vl is vlen/8. + +#contraction_accesses_rvv_vfwmacc = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_vfwmacc_f16_vlen256( + %lhs: vector<1x1xf16>, %rhs: vector<32x1xf16>, %acc: vector<1x32xf32>) + -> vector<1x32xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses_rvv_vfwmacc, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + > : vector<1x1xf16>, vector<32x1xf16> into vector<1x32xf32> + return %0 : vector<1x32xf32> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_vfwmacc_f16_vlen256 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: %[[VL:.+]] = arith.constant 32 : i64 +// CHECK-NOT: arith.extf +// CHECK: vector.extract {{.*}} : f16 from vector<{{1x1|1}}xf16> +// CHECK-NOT: vector.broadcast {{.*}} : f16 to vector<[8]xf16> +// CHECK: llvm.call_intrinsic "llvm.riscv.vfwmacc"({{.*}}, %[[VL]], %[[POL]]) : (vector<[8]xf32>, f16, vector<[8]xf16>, i64, i64, i64) -> vector<[8]xf32> + +// ----- + +// RISC-V VLEN=256 swapped vfwmacc: scalar from unit-N RHS. + +#contraction_accesses_rvv_vfwmacc_t = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_vfwmacc_f16_swapped_vlen256( + %lhs: vector<32x1xf16>, %rhs: vector<1x1xf16>, %acc: vector<32x1xf32>) + -> vector<32x1xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses_rvv_vfwmacc_t, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + > : vector<32x1xf16>, vector<1x1xf16> into vector<32x1xf32> + return %0 : vector<32x1xf32> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_vfwmacc_f16_swapped_vlen256 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: %[[VL:.+]] = arith.constant 32 : i64 +// CHECK-NOT: arith.extf +// CHECK: vector.extract {{.*}} : f16 from vector<{{1x1|1}}xf16> +// CHECK-NOT: vector.broadcast {{.*}} : f16 to vector<[8]xf16> +// CHECK: llvm.call_intrinsic "llvm.riscv.vfwmacc"({{.*}}, %[[VL]], %[[POL]]) : (vector<[8]xf32>, f16, vector<[8]xf16>, i64, i64, i64) -> vector<[8]xf32> + +// ----- + +// Zvfhmin f16 accumulator: extf inside distribute, truncf inside reassemble. + +#contraction_accesses_rvv_widenf32 = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_f16_widenf32_vlen256( + %lhs: vector<1x1xf16>, %rhs: vector<32x1xf16>, %acc: vector<1x32xf16>) + -> vector<1x32xf16> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) < + indexing_maps = #contraction_accesses_rvv_widenf32, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + > : vector<1x1xf16>, vector<32x1xf16> into vector<1x32xf16> + return %0 : vector<1x32xf16> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_f16_widenf32_vlen256 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: %[[VL:.+]] = arith.constant 32 : i64 +// CHECK: util.hoistable_conversion "data_tiled_acc_distribute" +// CHECK: arith.extf {{.*}} : vector<{{1x32|32}}xf16> to vector<{{1x32|32}}xf32> +// CHECK: vector.scalable.insert {{.*}} : vector<32xf32> into vector<[8]xf32> +// CHECK: arith.extf {{.*}} : vector<{{1x1|1}}xf16> to vector<{{1x1|1}}xf32> +// CHECK: arith.extf {{.*}} : vector<{{32x1|32}}xf16> to vector<{{32x1|32}}xf32 +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[VL]], %[[POL]]) : (vector<[8]xf32>, f32, vector<[8]xf32>, i64, i64, i64) -> vector<[8]xf32> +// CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" +// CHECK: vector.scalable.extract {{.*}} : vector<32xf32> from vector<[8]xf32> +// CHECK: arith.truncf {{.*}} : vector<32xf32> to vector<32xf16>
diff --git a/compiler/src/iree/compiler/Codegen/ExternalInterfaces/CPUEncodingExternalModels.cpp b/compiler/src/iree/compiler/Codegen/ExternalInterfaces/CPUEncodingExternalModels.cpp index 1261a1f..ee313ac 100644 --- a/compiler/src/iree/compiler/Codegen/ExternalInterfaces/CPUEncodingExternalModels.cpp +++ b/compiler/src/iree/compiler/Codegen/ExternalInterfaces/CPUEncodingExternalModels.cpp
@@ -437,7 +437,14 @@ return {"+v"}; case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16: case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16: + case MMAIntrinsic::MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16: + case MMAIntrinsic::MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16: return {"+v", "+zvfh"}; + case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32: + return {"+v", "+zvfhmin"}; default: return {}; } @@ -569,8 +576,13 @@ if (required.empty()) { continue; } - if (llvm::all_of(required, - [&](StringRef f) { return hasFeature(config, f); })) { + if (llvm::all_of(required, [&](StringRef f) { + if (hasFeature(config, f)) { + return true; + } + // Zvfh includes Zvfhmin; IREE matches feature strings exactly. + return f == "+zvfhmin" && hasFeature(config, "+zvfh"); + })) { out.push_back(intr); } } @@ -624,6 +636,12 @@ MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32, MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16, MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16, + MMAIntrinsic::MMA_RISCV_V_VFWMACC_1x8VLsx1_F32_F16, + MMAIntrinsic::MMA_RISCV_V_VFWMACC_8VLsx1x1_F32_F16, + MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32, + MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32, + MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32, + MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32, }; checkIntrinsicRequiredFeatures(config, kAllRiscvV, out); } @@ -685,6 +703,29 @@ info.lhsBits = lhsTy.getIntOrFloatBitWidth(); info.rhsBits = rhsTy.getIntOrFloatBitWidth(); info.accBits = accTy.getIntOrFloatBitWidth(); + // The vector operand is extf'd to f32 before vfmacc.vf, so it occupies an + // LMUL=4 group. WIDENF32 also keeps the accumulator at f32 across K. + bool billVectorAsF32 = false; + switch (intr) { + case IREE::CPU::MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F16_CASTF32: + case IREE::CPU::MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F16_CASTF32: + billVectorAsF32 = true; + break; + case IREE::CPU::MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16_WIDENF32: + case IREE::CPU::MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16_WIDENF32: + billVectorAsF32 = true; + info.accBits = 32; + break; + default: + break; + } + if (billVectorAsF32) { + if (IREE::CPU::isMNSwapped(intr)) { + info.lhsBits = 32; + } else { + info.rhsBits = 32; + } + } return info; }
diff --git a/tests/e2e/matmul/BUILD.bazel b/tests/e2e/matmul/BUILD.bazel index 749402c..11cbd1b 100644 --- a/tests/e2e/matmul/BUILD.bazel +++ b/tests/e2e/matmul/BUILD.bazel
@@ -296,6 +296,86 @@ test_type = "matmul", ) +iree_generated_e2e_runner_test( + name = "e2e_matmul_cpu_dt_inner_tiled_riscv_small_f16_f16", + compiler_flags = [ + "--iree-opt-data-tiling", + "--iree-llvmcpu-enable-inner-tiled", + ], + generator = ":generate_e2e_matmul_tests", + generator_args = [ + "--lhs_rhs_type=f16", + "--acc_type=f16", + "--shapes=small", + ], + target_backends_and_drivers = [ + ("llvm-cpu", "local-task"), + ], + target_cpu_features_variants = ["riscv_64:rvv:+m,+v,+zvl512b,+zvfh"], + test_runner = "//tools/testing/e2e:iree-e2e-matmul-test", + test_type = "matmul", +) + +iree_generated_e2e_runner_test( + name = "e2e_matmul_cpu_dt_inner_tiled_riscv_small_f16_f32", + compiler_flags = [ + "--iree-opt-data-tiling", + "--iree-llvmcpu-enable-inner-tiled", + ], + generator = ":generate_e2e_matmul_tests", + generator_args = [ + "--lhs_rhs_type=f16", + "--acc_type=f32", + "--shapes=small", + ], + target_backends_and_drivers = [ + ("llvm-cpu", "local-task"), + ], + target_cpu_features_variants = ["riscv_64:rvv:+m,+v,+zvl512b,+zvfhmin"], + test_runner = "//tools/testing/e2e:iree-e2e-matmul-test", + test_type = "matmul", +) + +iree_generated_e2e_runner_test( + name = "e2e_matmul_cpu_dt_inner_tiled_riscv_small_f16_f32_zvfh", + compiler_flags = [ + "--iree-opt-data-tiling", + "--iree-llvmcpu-enable-inner-tiled", + ], + generator = ":generate_e2e_matmul_tests", + generator_args = [ + "--lhs_rhs_type=f16", + "--acc_type=f32", + "--shapes=small", + ], + target_backends_and_drivers = [ + ("llvm-cpu", "local-task"), + ], + target_cpu_features_variants = ["riscv_64:rvv:+m,+v,+zvl512b,+zvfh"], + test_runner = "//tools/testing/e2e:iree-e2e-matmul-test", + test_type = "matmul", +) + +iree_generated_e2e_runner_test( + name = "e2e_matmul_cpu_dt_inner_tiled_riscv_small_f16_f16_zvfhmin", + compiler_flags = [ + "--iree-opt-data-tiling", + "--iree-llvmcpu-enable-inner-tiled", + ], + generator = ":generate_e2e_matmul_tests", + generator_args = [ + "--lhs_rhs_type=f16", + "--acc_type=f16", + "--shapes=small", + ], + target_backends_and_drivers = [ + ("llvm-cpu", "local-task"), + ], + target_cpu_features_variants = ["riscv_64:rvv:+m,+v,+zvl512b,+zvfhmin"], + test_runner = "//tools/testing/e2e:iree-e2e-matmul-test", + test_type = "matmul", +) + # LLVMCPU, data-tiling lowering to iree_codegen.inner_tiled AND routing the # inner_tiled op through a built-in C-bitcode microkernel via # `--iree-llvmcpu-enable-llvm-ukernels=inner_tiled`. Restricted to the
diff --git a/tests/e2e/matmul/CMakeLists.txt b/tests/e2e/matmul/CMakeLists.txt index e790013..4d103f8 100644 --- a/tests/e2e/matmul/CMakeLists.txt +++ b/tests/e2e/matmul/CMakeLists.txt
@@ -742,6 +742,102 @@ iree_generated_e2e_runner_test( NAME + e2e_matmul_cpu_dt_inner_tiled_riscv_small_f16_f16 + TEST_TYPE + matmul + GENERATOR + "generate_e2e_matmul_tests.py" + GENERATOR_ARGS + "--lhs_rhs_type=f16" + "--acc_type=f16" + "--shapes=small" + TEST_RUNNER + iree_tools_testing_e2e_iree-e2e-matmul-test + TARGET_BACKENDS + "llvm-cpu" + DRIVERS + "local-task" + COMPILER_FLAGS + "--iree-opt-data-tiling" + "--iree-llvmcpu-enable-inner-tiled" + TARGET_CPU_FEATURES_VARIANTS + "riscv_64:rvv:+m,+v,+zvl512b,+zvfh" +) + +iree_generated_e2e_runner_test( + NAME + e2e_matmul_cpu_dt_inner_tiled_riscv_small_f16_f32 + TEST_TYPE + matmul + GENERATOR + "generate_e2e_matmul_tests.py" + GENERATOR_ARGS + "--lhs_rhs_type=f16" + "--acc_type=f32" + "--shapes=small" + TEST_RUNNER + iree_tools_testing_e2e_iree-e2e-matmul-test + TARGET_BACKENDS + "llvm-cpu" + DRIVERS + "local-task" + COMPILER_FLAGS + "--iree-opt-data-tiling" + "--iree-llvmcpu-enable-inner-tiled" + TARGET_CPU_FEATURES_VARIANTS + "riscv_64:rvv:+m,+v,+zvl512b,+zvfhmin" +) + +iree_generated_e2e_runner_test( + NAME + e2e_matmul_cpu_dt_inner_tiled_riscv_small_f16_f32_zvfh + TEST_TYPE + matmul + GENERATOR + "generate_e2e_matmul_tests.py" + GENERATOR_ARGS + "--lhs_rhs_type=f16" + "--acc_type=f32" + "--shapes=small" + TEST_RUNNER + iree_tools_testing_e2e_iree-e2e-matmul-test + TARGET_BACKENDS + "llvm-cpu" + DRIVERS + "local-task" + COMPILER_FLAGS + "--iree-opt-data-tiling" + "--iree-llvmcpu-enable-inner-tiled" + TARGET_CPU_FEATURES_VARIANTS + "riscv_64:rvv:+m,+v,+zvl512b,+zvfh" +) + +iree_generated_e2e_runner_test( + NAME + e2e_matmul_cpu_dt_inner_tiled_riscv_small_f16_f16_zvfhmin + TEST_TYPE + matmul + GENERATOR + "generate_e2e_matmul_tests.py" + GENERATOR_ARGS + "--lhs_rhs_type=f16" + "--acc_type=f16" + "--shapes=small" + TEST_RUNNER + iree_tools_testing_e2e_iree-e2e-matmul-test + TARGET_BACKENDS + "llvm-cpu" + DRIVERS + "local-task" + COMPILER_FLAGS + "--iree-opt-data-tiling" + "--iree-llvmcpu-enable-inner-tiled" + TARGET_CPU_FEATURES_VARIANTS + "riscv_64:rvv:+m,+v,+zvl512b,+zvfhmin" +) + +iree_generated_e2e_runner_test( + NAME e2e_matmul_cpu_dt_inner_tiled_llvm_ukernel_bf16_f32 TEST_TYPE matmul