[Codegen][CPU] Add RISC-V V f32 vfmacc.vf inner_tiled MMA. (#24933) ## Summary Lowers RISC-V V f32 `inner_tiled` MMA (`MMA_RISCV_V_VFMACC_*_F32_F32`) to `llvm.riscv.vfmacc` (`vfmacc.vf`). Tile N is `vlen/8`. Introduces shared helper `lowerRiscvVFmaccLike`. This is the first stacked dtype PR from #24926 review. Performance will be tracked here #24932 Assisted-by: Cursor Towards: #24311 --------- Signed-off-by: Terence <terrencemahlatini@gmail.com> Co-authored-by: Cursor <cursoragent@cursor.com>
diff --git a/compiler/src/iree/compiler/Codegen/Common/test/materialize_encoding_riscv_inner_tiled.mlir b/compiler/src/iree/compiler/Codegen/Common/test/materialize_encoding_riscv_inner_tiled.mlir index 2ef3a35..e5ade49 100644 --- a/compiler/src/iree/compiler/Codegen/Common/test/materialize_encoding_riscv_inner_tiled.mlir +++ b/compiler/src/iree/compiler/Codegen/Common/test/materialize_encoding_riscv_inner_tiled.mlir
@@ -111,3 +111,173 @@ // CHECK: iree_codegen.inner_tiled // CHECK-SAME: intrinsic = MMA_GENERIC_SCALAR_1x1x1_REG16 // CHECK-NOT: vlen + +// ----- + +// f32 vfmacc.vf. N tile is vlen/8; intrinsics_m stays 6. +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +func.func @matmul_f32_zvl128b(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?x?xf32>, %m: index, %n: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvl128b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf32> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf32> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %n, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %n, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf32, #lhs>, tensor<?x?xf32, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %n, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f32_zvl128b( +// CHECK: %[[PACK_LHS:.+]] = linalg.pack {{.*}}inner_tiles = [6, 1] +// CHECK-SAME: -> tensor<?x?x6x1xf32> +// CHECK: %[[EXPANDED:.+]] = tensor.expand_shape %[[PACK_LHS]] {{\[}}[0], [1], [2], [3, 4]{{\]}} +// CHECK-SAME: tensor<?x?x6x1xf32> into tensor<?x?x6x1x1xf32> +// CHECK: %[[PACK_RHS:.+]] = linalg.pack {{.*}}inner_tiles = [16, 1] +// CHECK-SAME: -> tensor<?x?x16x1xf32> +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled ins(%[[EXPANDED]], %[[PACK_RHS]]) +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, intrinsics_m = 6, vlen = 128> +// CHECK-SAME: tensor<?x?x6x1x1xf32>, tensor<?x?x16x1xf32> into tensor<?x?x6x1x16xf32> + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +func.func @matmul_f32_zvl256b(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?x?xf32>, %m: index, %n: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvl256b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf32> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf32> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %n, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %n, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf32, #lhs>, tensor<?x?xf32, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %n, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f32_zvl256b( +// CHECK: %[[PACK_RHS:.+]] = linalg.pack {{.*}}inner_tiles = [32, 1] +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, intrinsics_m = 6, vlen = 256> + +// ----- + +// Narrow-N: with `iteration_sizes = [?, 8, ?]` the natural 1x8VLsx1 wastes +// all but 8 of its 32 N lanes, so the cost model picks the M↔N-swapped +// 8VLsx1x1, which puts the 32 lanes on M. +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, 8, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, 8, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, 8, ?]> +func.func @matmul_f32_zvl256b_narrow_n(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?x?xf32>, %m: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvl256b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf32> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf32> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf32, #lhs>, tensor<?x?xf32, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f32_zvl256b_narrow_n( +// CHECK: %[[PACK_LHS:.+]] = linalg.pack {{.*}}inner_tiles = [32, 1] +// CHECK-SAME: -> tensor<?x?x32x1xf32> +// CHECK: %[[PACK_RHS:.+]] = linalg.pack {{.*}}inner_tiles = [6, 1] +// CHECK-SAME: -> tensor<?x?x6x1xf32> +// CHECK: %[[EXPANDED:.+]] = tensor.expand_shape %[[PACK_RHS]] {{\[}}[0], [1], [2], [3, 4]{{\]}} +// CHECK-SAME: tensor<?x?x6x1xf32> into tensor<?x?x6x1x1xf32> +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled ins(%[[PACK_LHS]], %[[EXPANDED]]) +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32, intrinsics_n = 6, vlen = 256> +// CHECK-SAME: tensor<?x?x32x1xf32>, tensor<?x?x6x1x1xf32> into tensor<?x?x6x32x1xf32> +// CHECK: %[[TRANSPOSED:.+]] = linalg.transpose ins(%[[INNER]] +// CHECK: %[[COLLAPSED:.+]] = tensor.collapse_shape %[[TRANSPOSED]] +// CHECK-SAME: into tensor<?x?x32x6xf32> +// CHECK: linalg.unpack %[[COLLAPSED]] {{.*}}inner_tiles = [32, 6] + +// ----- + +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +func.func @matmul_f32_zvl512b(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?x?xf32>, %m: index, %n: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v,+zvl512b", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf32> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf32> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %n, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %n, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf32, #lhs>, tensor<?x?xf32, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %n, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f32_zvl512b( +// CHECK: %[[PACK_RHS:.+]] = linalg.pack {{.*}}inner_tiles = [64, 1] +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, intrinsics_m = 6, vlen = 512> + +// ----- + +// Plain +v falls back to VLEN=128. +#map = affine_map<(d0, d1, d2) -> (d0, d2)> +#map1 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map2 = affine_map<(d0, d1, d2) -> (d0, d1)> +#lhs = #iree_encoding.encoding<operand_index = 0, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#rhs = #iree_encoding.encoding<operand_index = 1, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +#acc = #iree_encoding.encoding<operand_index = 2, op_type = matmul, element_types = [f32, f32, f32], user_indexing_maps = [#map, #map1, #map2], iteration_sizes = [?, ?, ?]> +func.func @matmul_f32_plain_v(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?x?xf32>, %m: index, %n: index, %k: index) -> tensor<?x?xf32> attributes { + hal.executable.target = #hal.executable.target<"llvm-cpu", "embedded-elf-riscv_64", {target_triple = "riscv64-unknown-unknown-eabi-elf", cpu_features = "+m,+a,+f,+d,+c,+v", enable_inner_tiled = true, iree.encoding.resolver = #iree_cpu.cpu_encoding_resolver<>}> +} { + %c0 = arith.constant 0 : index + %c1 = arith.constant 1 : index + %cst = arith.constant 0.0 : f32 + %d0 = tensor.dim %arg0, %c0 : tensor<?x?xf32> + %d1 = tensor.dim %arg1, %c1 : tensor<?x?xf32> + %0 = iree_encoding.set_encoding %arg0 encoding_dims{%m, %n, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #lhs> + %1 = iree_encoding.set_encoding %arg1 encoding_dims{%m, %n, %k} : tensor<?x?xf32> -> tensor<?x?xf32, #rhs> + %2 = tensor.empty(%d0, %d1) : tensor<?x?xf32, #acc> + %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %4 = linalg.matmul ins(%0, %1 : tensor<?x?xf32, #lhs>, tensor<?x?xf32, #rhs>) + outs(%3 : tensor<?x?xf32, #acc>) -> tensor<?x?xf32, #acc> + %5 = iree_encoding.unset_encoding %4 encoding_dims{%m, %n, %k} : tensor<?x?xf32, #acc> -> tensor<?x?xf32>{%d0, %d1} + return %5 : tensor<?x?xf32> +} +// CHECK-LABEL: func @matmul_f32_plain_v( +// CHECK: %[[INNER:.+]] = iree_codegen.inner_tiled +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, intrinsics_m = 6, vlen = 128>
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/BUILD.bazel b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/BUILD.bazel index ac9b672..ea27df1 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/BUILD.bazel +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/BUILD.bazel
@@ -78,6 +78,7 @@ "@llvm-project//mlir:Support", "@llvm-project//mlir:TensorDialect", "@llvm-project//mlir:TilingInterface", + "@llvm-project//mlir:UBDialect", "@llvm-project//mlir:VectorDialect", ], )
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/CMakeLists.txt index ea6059e..5527f02 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/CMakeLists.txt +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/CMakeLists.txt
@@ -41,6 +41,7 @@ MLIRSupport MLIRTensorDialect MLIRTilingInterface + MLIRUBDialect MLIRVectorDialect iree::compiler::Codegen::Dialect::Codegen::IR::IREECodegenDialect iree::compiler::Codegen::Dialect::Codegen::Utils
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUAttrs.cpp b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUAttrs.cpp index 058946b..2d13e54 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUAttrs.cpp +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUAttrs.cpp
@@ -19,6 +19,7 @@ #include "mlir/Dialect/Linalg/IR/Linalg.h" #include "mlir/Dialect/Linalg/IR/LinalgInterfaces.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" +#include "mlir/Dialect/UB/IR/UBOps.h" #include "mlir/Dialect/Utils/IndexingUtils.h" #include "mlir/Dialect/Vector/IR/VectorOps.h" #include "mlir/IR/AsmState.h" @@ -485,13 +486,13 @@ return Tuple{1, 4, 1}; case MMAIntrinsic::MMA_ARM_SVE_FMLA_4VLx1x1_F32_F32: return Tuple{4, 1, 1}; - // VLs = vlen / 8 lanes, one LMUL=2 register group at 16-bit elements. + // RVV: M or N is vlen/8. Enum bit 0 is the swapped orientation. + case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32: case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16: case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16: { int64_t vl = vlen / 8; - bool transposed = - intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16; - return transposed ? Tuple{vl, 1, 1} : Tuple{1, vl, 1}; + return isMNSwapped(intrinsic) ? Tuple{vl, 1, 1} : Tuple{1, vl, 1}; } default: if (isGenericScalar(intrinsic)) { @@ -514,6 +515,11 @@ constexpr uint32_t kMMAIntrinsicISAX86Avx512 = 0x1300; constexpr uint32_t kMMAIntrinsicISAArmSve = 0x2200; constexpr uint32_t kMMAIntrinsicISARiscvV = 0x3100; +constexpr uint32_t kMMAIntrinsicMNSwappedBit = 0x0001; + +bool isMNSwapped(MMAIntrinsic intr) { + return (static_cast<uint32_t>(intr) & kMMAIntrinsicMNSwappedBit) != 0; +} bool isGenericScalar(MMAIntrinsic intr) { return (static_cast<uint32_t>(intr) & kMMAIntrinsicISAMask) == @@ -737,6 +743,8 @@ return {f64, f64, f64}; case MMAIntrinsic::MMA_X86_AVX512_1x16x1_F32_F32: case MMAIntrinsic::MMA_X86_AVX512_16x1x1_F32_F32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32: return {f32, f32, f32}; case MMAIntrinsic::MMA_X86_AVX512_1x16x1_F32_F16_CASTF32: case MMAIntrinsic::MMA_X86_AVX512_16x1x1_F32_F16_CASTF32: @@ -962,17 +970,104 @@ return result; } +// Insert a fixed 1-D vector into poison `nxv8` so 8 * vscale == vlen/8 lanes. +// Poison plus a tail-agnostic policy lets LLVM treat the insert as a +// register copy. Already-scalable values are returned unchanged. +static Value insertFixedIntoNxv8(OpBuilder &b, Location loc, Value v) { + auto vt = dyn_cast<VectorType>(v.getType()); + if (!vt) { + return v; + } + if (vt.getRank() > 1) { + v = vector::ShapeCastOp::create( + b, loc, VectorType::get({vt.getNumElements()}, vt.getElementType()), v); + vt = cast<VectorType>(v.getType()); + } + if (vt.isScalable()) { + return v; + } + auto scalableTy = VectorType::get({8}, vt.getElementType(), {true}); + Value pad = ub::PoisonOp::create(b, loc, scalableTy); + return vector::ScalableInsertOp::create(b, loc, v, pad, /*pos=*/0); +} + +static Value extractNxv8ToFixed(OpBuilder &b, Location loc, Value v, + int64_t lanes) { + auto vt = cast<VectorType>(v.getType()); + if (!vt.isScalable()) { + return v; + } + auto fixedTy = VectorType::get({lanes}, vt.getElementType()); + return vector::ScalableExtractOp::create(b, loc, fixedTy, v, /*pos=*/0); +} + +// RVV `.vf` FMA (`vfmacc` / `vfwmaccbf16`). Enum bit 0 selects which operand +// is the scalar (swapped => RHS). The vector operand is inserted into nxv8 so +// 8 * vscale == vlen/8 lanes. ACC is left scalable when the caller already +// legalized it (hoisted out of K). frm=7 is DYN; policy=1 is tail-agnostic. +static Value lowerRiscvVFmaccLike(OpBuilder &b, Location loc, + MMAIntrinsic intrinsic, int64_t vlen, + Value lhs, Value rhs, Value acc, + StringRef llvmName) { + bool lhsIsScalar = !isMNSwapped(intrinsic); + Value scalarSrc = lhsIsScalar ? lhs : rhs; + Value vec = lhsIsScalar ? rhs : lhs; + + auto flattenIfNeeded = [&](Value v) -> Value { + auto vt = dyn_cast<VectorType>(v.getType()); + if (!vt || vt.getRank() <= 1) { + return v; + } + return vector::ShapeCastOp::create( + b, loc, VectorType::get({vt.getNumElements()}, vt.getElementType()), v); + }; + vec = flattenIfNeeded(vec); + acc = flattenIfNeeded(acc); + scalarSrc = flattenIfNeeded(scalarSrc); + + auto scalarTy = cast<VectorType>(scalarSrc.getType()); + SmallVector<int64_t> zeros(scalarTy.getRank(), 0); + Value scalar = vector::ExtractOp::create(b, loc, scalarSrc, zeros); + + Value accScalable = insertFixedIntoNxv8(b, loc, acc); + Value vecScalable = insertFixedIntoNxv8(b, loc, vec); + bool accWasScalable = cast<VectorType>(acc.getType()).isScalable(); + + int64_t lanes = vlen / 8; + Value frm = arith::ConstantOp::create(b, loc, b.getI64IntegerAttr(7)); + Value vl = arith::ConstantOp::create(b, loc, b.getI64IntegerAttr(lanes)); + // RISC-V vector policy: bit0 = tail (0=tu, 1=ta), bit1 = mask (0=mu, 1=ma). + // 1 = tail-agnostic. The intrinsic is unmasked, so the mask bit stays 0. + Value policy = arith::ConstantOp::create(b, loc, b.getI64IntegerAttr(1)); + Value resultScalable = + LLVM::CallIntrinsicOp::create( + b, loc, accScalable.getType(), b.getStringAttr(llvmName), + ValueRange{accScalable, scalar, vecScalable, frm, vl, policy}) + .getResult(0); + if (accWasScalable) { + return resultScalable; + } + auto accFixedTy = cast<VectorType>(acc.getType()); + return vector::ScalableExtractOp::create(b, loc, accFixedTy, resultScalable, + /*pos=*/0); +} + // Lowers a MMAIntrinsic to a llvm.call_intrinsic op, plus any necessary // additional ops (potentially broadcasting or widening LHS/RHS or creating an // add op if the intrinsic isn't already adding the accumulator). static Value createCpuMmaIntrinsicCall(OpBuilder &builder, Location loc, MMAIntrinsic intrinsic, Value lhs, - Value rhs, Value acc) { + Value rhs, Value acc, int64_t vlen) { // The 16x16x2 i8 intrinsic processes whole panels and has its own widen / // shuffle scheme; it bypasses the per-row widen + broadcast path below. if (intrinsic == MMAIntrinsic::MMA_X86_AVX512VNNI_16x16x2_I32_I8_CASTI16) { return lowerX86Avx512Vnni16x16x2I8(builder, loc, lhs, rhs, acc); } + if (intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32) { + return lowerRiscvVFmaccLike(builder, loc, intrinsic, vlen, lhs, rhs, acc, + "llvm.riscv.vfmacc"); + } // Sign-/float-extend a vector to a wider element type. Used by the // *_CASTF32 (f16 → f32) and *_CASTI16 (i8 → i16) variants where the // intrinsic only exists at the wider type. @@ -1255,8 +1350,25 @@ } auto emitIntrinsic = [&](OpBuilder &b, Location loc, Value lhs, Value rhs, Value acc) -> Value { - return createCpuMmaIntrinsicCall(b, loc, intrinsic, lhs, rhs, acc); + return createCpuMmaIntrinsicCall(b, loc, intrinsic, lhs, rhs, acc, + getVlen()); }; + if (intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32 || + intrinsic == MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32) { + int64_t lanes = getVlen() / 8; + auto legalizeAcc = [](OpBuilder &b, Location loc, Value v) -> Value { + return insertFixedIntoNxv8(b, loc, v); + }; + auto restoreAcc = [lanes](OpBuilder &b, Location loc, Value v) -> Value { + return extractNxv8ToFixed(b, loc, v, lanes); + }; + return Codegen::buildDataTiledMMAUnderlyingOperations( + builder, loc, getSwizzle(*this, /*operandIdx=*/0), + getSwizzle(*this, /*operandIdx=*/1), + getSwizzle(*this, /*operandIdx=*/2), getIntrinsicsM(), getIntrinsicsN(), + getIntrinsicsK(), inputs, outputs, emitIntrinsic, results, legalizeAcc, + restoreAcc); + } return Codegen::buildDataTiledMMAUnderlyingOperations( builder, loc, getSwizzle(*this, /*operandIdx=*/0), getSwizzle(*this, /*operandIdx=*/1), getSwizzle(*this, /*operandIdx=*/2),
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.cpp b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.cpp index 941aff5..22a7987 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.cpp +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.cpp
@@ -6,9 +6,11 @@ #include "iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.h" -#include "iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.cpp.inc" #include "iree/compiler/Codegen/Dialect/CPU/IR/IREECPUTypes.h" #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenDialect.h" +#include "mlir/Dialect/UB/IR/UBOps.h" + +#include "iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.cpp.inc" namespace mlir::iree_compiler::IREE::CPU {
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.td b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.td index 6c77cfb..038179c 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.td +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUDialect.td
@@ -25,6 +25,9 @@ LLVM are handled separately. }]; let useDefaultAttributePrinterParser = 1; + // `DataTiledMMAAttr::buildUnderlyingOperations` creates `ub.poison` when + // lowering RVV intrinsics. + let dependentDialects = ["::mlir::ub::UBDialect"]; let extraClassDeclaration = [{ void registerAttributes();
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUEnums.td b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUEnums.td index 84ed106..c730ae1 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUEnums.td +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUEnums.td
@@ -174,6 +174,13 @@ def MMA_ARM_SVE_FMLA_4VLx1x1_F32_F32 : I32EnumAttrCase<"MMA_ARM_SVE_FMLA_4VLx1x1_F32_F32", 0x2211>; +// RISC-V V `vfmacc.vf` (f32, f32). 1x8VLsx1 or swapped 8VLsx1x1. +// 8VLs = LMUL=4; lane count is DataTiledMMAAttr.vlen/8. Requires +v. +def MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32 + : I32EnumAttrCase<"MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32", 0x3110>; +def MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32 + : I32EnumAttrCase<"MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32", 0x3111>; + // RISC-V V `vfmacc.vf` (f16, f16): a 1x8VLsx1 matmul tile, or 8VLsx1x1 in the // M<->N-swapped orientation. `8VLs` is 8 f16 elements per 64-bit vector-length // unit, i.e. one LMUL=2 register group, and s stands for static. The LHS is a @@ -233,6 +240,8 @@ MMA_ARM_SVE_FMLA_1x4VLx1_F32_F32, MMA_ARM_SVE_FMLA_4VLx1x1_F32_F32, // RISC-V V + MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, + MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32, MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16, MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16,
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUTypes.h b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUTypes.h index 50c2140..ad9db3e 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUTypes.h +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/IREECPUTypes.h
@@ -89,6 +89,10 @@ // RISC-V family of intrinsics. bool isVlenParameterized(MMAIntrinsic intr); +// True if `intr` is the M↔N-swapped orientation of a natural/swapped pair, +// i.e. bit 0 of the enum value is set (see IREECPUEnums.td). +bool isMNSwapped(MMAIntrinsic intr); + // For an `MMA_GENERIC_SCALAR_1x1x1_REG*` intrinsic, returns the register // budget encoded in the enum case (8 or 16). Asserts otherwise. int64_t getGenericScalarRegisterBudget(MMAIntrinsic intr);
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/iree_cpu_inner_tiled_ops.mlir b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/iree_cpu_inner_tiled_ops.mlir index 45ea76c..3a33880 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/iree_cpu_inner_tiled_ops.mlir +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/iree_cpu_inner_tiled_ops.mlir
@@ -327,3 +327,52 @@ // CHECK-LABEL: func @cpu_riscv_v_vfmacc_f16_swapped // CHECK: iree_codegen.inner_tiled ins(%arg0, %arg1) outs(%arg2) // CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16, vlen = 512> +// CHECK-SAME: semantics = #iree_cpu.mma_semantics<> + +// ----- + +#contraction_accesses = [ + affine_map<(i, j, k) -> (i, k)>, + affine_map<(i, j, k) -> (k, j)>, + affine_map<(i, j, k) -> (i, j)> +] +// MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32 at vlen = 256, intrinsics 1x1x1 +func.func @cpu_riscv_v_vfmacc_f32( + %lhs: vector<1x1x1xf32>, %rhs: vector<1x1x32xf32>, %acc: vector<1x1x32xf32>) + -> vector<1x1x32xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) { + indexing_maps = #contraction_accesses, + iterator_types = [#linalg.iterator_type<parallel>, #linalg.iterator_type<parallel>, #linalg.iterator_type<reduction>], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + } : vector<1x1x1xf32>, vector<1x1x32xf32> into vector<1x1x32xf32> + return %0 : vector<1x1x32xf32> +} +// CHECK-LABEL: func @cpu_riscv_v_vfmacc_f32 +// CHECK: iree_codegen.inner_tiled ins(%arg0, %arg1) outs(%arg2) +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, vlen = 256> +// CHECK-SAME: semantics = #iree_cpu.mma_semantics<> + +// ----- + +#contraction_accesses = [ + affine_map<(i, j, k) -> (i, k)>, + affine_map<(i, j, k) -> (k, j)>, + affine_map<(i, j, k) -> (i, j)> +] +// MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32 at vlen = 128, intrinsics 1x1x1 +func.func @cpu_riscv_v_vfmacc_f32_swapped( + %lhs: vector<1x1x16xf32>, %rhs: vector<1x1x1xf32>, %acc: vector<1x1x16xf32>) + -> vector<1x1x16xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) { + indexing_maps = #contraction_accesses, + iterator_types = [#linalg.iterator_type<parallel>, #linalg.iterator_type<parallel>, #linalg.iterator_type<reduction>], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32, vlen = 128>, + semantics = #iree_cpu.mma_semantics<> + } : vector<1x1x16xf32>, vector<1x1x1xf32> into vector<1x1x16xf32> + return %0 : vector<1x1x16xf32> +} +// CHECK-LABEL: func @cpu_riscv_v_vfmacc_f32_swapped +// CHECK: iree_codegen.inner_tiled ins(%arg0, %arg1) outs(%arg2) +// CHECK-SAME: kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32, vlen = 128> +// CHECK-SAME: semantics = #iree_cpu.mma_semantics<>
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/lower_inner_tiled.mlir b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/lower_inner_tiled.mlir index 9c6def0..098cbb0 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/lower_inner_tiled.mlir +++ b/compiler/src/iree/compiler/Codegen/Dialect/CPU/IR/test/lower_inner_tiled.mlir
@@ -419,3 +419,149 @@ // CHECK-DAG: vector.shuffle %{{.+}} [0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12] // CHECK-DAG: vector.shuffle %{{.+}} [0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3] // CHECK-COUNT-16: llvm.call_intrinsic "llvm.x86.avx512.vpdpwssd.512" + +// ----- + +// RISC-V VLEN=256 1×32×1 f32: scalar .vf, not splat. + +#contraction_accesses_rvv = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_1x8vlsx1_f32_vlen256( + %lhs: vector<1x1xf32>, %rhs: vector<32x1xf32>, %acc: vector<1x32xf32>) + -> vector<1x32xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) { + indexing_maps = #contraction_accesses_rvv, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + } : vector<1x1xf32>, vector<32x1xf32> into vector<1x32xf32> + return %0 : vector<1x32xf32> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_1x8vlsx1_f32_vlen256 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: ub.poison : vector<[8]xf32> +// CHECK: util.hoistable_conversion "data_tiled_acc_distribute" +// CHECK: vector.scalable.insert {{.*}} : vector<32xf32> into vector<[8]xf32> +// CHECK: vector.extract {{.*}} : f32 from vector<{{1x1|1}}xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<32xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<[8]xf32> +// CHECK: vector.scalable.insert {{.*}} : vector<32xf32> into vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[POL]]) : (vector<[8]xf32>, f32, vector<[8]xf32>, i64, i64, i64) -> vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" +// CHECK: vector.scalable.extract {{.*}} : vector<32xf32> from vector<[8]xf32> + +// ----- + +// RISC-V VLEN=128: lane count tracks vlen/8. + +#contraction_accesses_rvv128 = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_1x8vlsx1_f32_vlen128( + %lhs: vector<1x1xf32>, %rhs: vector<16x1xf32>, %acc: vector<1x16xf32>) + -> vector<1x16xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) { + indexing_maps = #contraction_accesses_rvv128, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, vlen = 128>, + semantics = #iree_cpu.mma_semantics<> + } : vector<1x1xf32>, vector<16x1xf32> into vector<1x16xf32> + return %0 : vector<1x16xf32> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_1x8vlsx1_f32_vlen128 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: ub.poison : vector<[8]xf32> +// CHECK: util.hoistable_conversion "data_tiled_acc_distribute" +// CHECK: vector.scalable.insert {{.*}} : vector<16xf32> into vector<[8]xf32> +// CHECK: vector.extract {{.*}} : f32 from vector<{{1x1|1}}xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<[8]xf32> +// CHECK: vector.scalable.insert {{.*}} : vector<16xf32> into vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[POL]]) : (vector<[8]xf32>, f32, vector<[8]xf32>, i64, i64, i64) -> vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" +// CHECK: vector.scalable.extract {{.*}} : vector<16xf32> from vector<[8]xf32> + +// ----- + +// RISC-V VLEN=256 32×1×1 f32: scalar from unit-N RHS. + +#contraction_accesses_rvv_t = [ + affine_map<() -> ()>, + affine_map<() -> ()>, + affine_map<() -> ()> +] +func.func @lower_riscv_v_8vlsx1x1_f32_vlen256( + %lhs: vector<32x1xf32>, %rhs: vector<1x1xf32>, %acc: vector<32x1xf32>) + -> vector<32x1xf32> { + %0 = iree_codegen.inner_tiled ins(%lhs, %rhs) outs(%acc) { + indexing_maps = #contraction_accesses_rvv_t, + iterator_types = [], + kind = #iree_cpu.data_tiled_mma_layout<intrinsic = MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32, vlen = 256>, + semantics = #iree_cpu.mma_semantics<> + } : vector<32x1xf32>, vector<1x1xf32> into vector<32x1xf32> + return %0 : vector<32x1xf32> +} + +module attributes { transform.with_named_sequence } { + transform.named_sequence @__transform_main(%root: !transform.any_op {transform.readonly}) { + %func = transform.structured.match ops{["func.func"]} in %root + : (!transform.any_op) -> !transform.any_op + transform.apply_patterns to %func { + transform.apply_patterns.iree.lower_inner_tiled + } : !transform.any_op + transform.yield + } +} + +// CHECK-LABEL: func @lower_riscv_v_8vlsx1x1_f32_vlen256 +// CHECK: %[[POL:.+]] = arith.constant 1 : i64 +// CHECK: ub.poison : vector<[8]xf32> +// CHECK: util.hoistable_conversion "data_tiled_acc_distribute" +// CHECK: vector.scalable.insert {{.*}} : vector<32xf32> into vector<[8]xf32> +// CHECK: vector.extract {{.*}} : f32 from vector<{{1x1|1}}xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<32xf32> +// CHECK-NOT: vector.broadcast {{.*}} : f32 to vector<[8]xf32> +// CHECK: vector.scalable.insert {{.*}} : vector<32xf32> into vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: llvm.call_intrinsic "llvm.riscv.vfmacc"({{.*}}, %[[POL]]) : (vector<[8]xf32>, f32, vector<[8]xf32>, i64, i64, i64) -> vector<[8]xf32> +// CHECK-NOT: vector.scalable.insert +// CHECK-NOT: vector.scalable.extract +// CHECK: util.hoistable_conversion "data_tiled_acc_reassemble" +// CHECK: vector.scalable.extract {{.*}} : vector<32xf32> from vector<[8]xf32>
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/Codegen/Utils/MMAUtils.cpp b/compiler/src/iree/compiler/Codegen/Dialect/Codegen/Utils/MMAUtils.cpp index 9e1c3d4..e13d577 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/Codegen/Utils/MMAUtils.cpp +++ b/compiler/src/iree/compiler/Codegen/Dialect/Codegen/Utils/MMAUtils.cpp
@@ -98,8 +98,8 @@ const TileSwizzle &rhsSwizzle, const TileSwizzle &accSwizzle, int64_t intrinsicsM, int64_t intrinsicsN, int64_t intrinsicsK, ValueRange inputs, ValueRange outputs, - DataTiledMMAIntrinsicEmitter emitIntrinsic, - SmallVectorImpl<Value> &results) { + DataTiledMMAIntrinsicEmitter emitIntrinsic, SmallVectorImpl<Value> &results, + DataTiledMMAValueXformer legalizeAcc, DataTiledMMAValueXformer restoreAcc) { if (inputs.size() != 2 || outputs.size() != 1) { return failure(); } @@ -123,8 +123,14 @@ [&](OpBuilder &b, Location loc, ValueRange args) -> SmallVector<Value> { Value accDistributed = reshapeToSwizzleDistributed(b, loc, args[0], accSwizzle); - return distributeMmaFragmentToIntrinsics(b, loc, accDistributed, - accSwizzle); + SmallVector<Value> pieces = distributeMmaFragmentToIntrinsics( + b, loc, accDistributed, accSwizzle); + if (legalizeAcc) { + for (Value &piece : pieces) { + piece = legalizeAcc(b, loc, piece); + } + } + return pieces; }); SmallVector<Value> intrinsicsAcc(distributeAccOp.getResults()); @@ -170,6 +176,9 @@ Value acc = arith::ConstantOp::create(b, loc, b.getZeroAttr(fullAccType)); for (Value intrAcc : args) { + if (restoreAcc) { + intrAcc = restoreAcc(b, loc, intrAcc); + } Value expandedAcc = vector::ShapeCastOp::create( b, loc, VectorType::get(accInternalShape, accElemType), intrAcc); acc = vector::InsertStridedSliceOp::create(b, loc, expandedAcc, acc,
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/Codegen/Utils/MMAUtils.h b/compiler/src/iree/compiler/Codegen/Dialect/Codegen/Utils/MMAUtils.h index d48bc93..b901395 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/Codegen/Utils/MMAUtils.h +++ b/compiler/src/iree/compiler/Codegen/Dialect/Codegen/Utils/MMAUtils.h
@@ -75,6 +75,13 @@ using DataTiledMMAIntrinsicEmitter = llvm::function_ref<Value( OpBuilder &builder, Location loc, Value lhs, Value rhs, Value acc)>; +/// Optional map applied to each per-intrinsic ACC piece inside the hoistable +/// distribute / reassemble pair. Empty means identity. Used by RVV to convert +/// fixed ACC rows to scalable `nxv8` (and back) so that conversion hoists out +/// of the K loop rather than running around every FMA. +using DataTiledMMAValueXformer = + llvm::function_ref<Value(OpBuilder &builder, Location loc, Value value)>; + /// Common body of `DataTiledMMAAttr::buildUnderlyingOperations`, used by both /// CPU and GPU. Distributes `inputs` and `outputs[0]` into per-intrinsic flat /// vectors using the three swizzles, runs an `(mu, nu, ku)` unroll loop @@ -88,13 +95,17 @@ /// Operand vector ranks may be either the swizzle's full expanded form (GPU) /// or the 2-D (outer × inner) collapsed form (CPU); the helper reshapes as /// needed. +/// +/// When `legalizeAcc` / `restoreAcc` are set they run inside the hoistable +/// distribute / reassemble bodies and must be inverses of each other. LogicalResult buildDataTiledMMAUnderlyingOperations( OpBuilder &builder, Location loc, const TileSwizzle &lhsSwizzle, const TileSwizzle &rhsSwizzle, const TileSwizzle &accSwizzle, int64_t intrinsicsM, int64_t intrinsicsN, int64_t intrinsicsK, ValueRange inputs, ValueRange outputs, - DataTiledMMAIntrinsicEmitter emitIntrinsic, - SmallVectorImpl<Value> &results); + DataTiledMMAIntrinsicEmitter emitIntrinsic, SmallVectorImpl<Value> &results, + DataTiledMMAValueXformer legalizeAcc = {}, + DataTiledMMAValueXformer restoreAcc = {}); } // namespace mlir::iree_compiler::IREE::Codegen
diff --git a/compiler/src/iree/compiler/Codegen/ExternalInterfaces/CPUEncodingExternalModels.cpp b/compiler/src/iree/compiler/Codegen/ExternalInterfaces/CPUEncodingExternalModels.cpp index 27561e0..1261a1f 100644 --- a/compiler/src/iree/compiler/Codegen/ExternalInterfaces/CPUEncodingExternalModels.cpp +++ b/compiler/src/iree/compiler/Codegen/ExternalInterfaces/CPUEncodingExternalModels.cpp
@@ -432,6 +432,9 @@ case MMAIntrinsic::MMA_X86_AVX512VNNI_16x1x4_I32_I8_UI8: case MMAIntrinsic::MMA_X86_AVX512VNNI_16x16x2_I32_I8_CASTI16: return {"+avx512vnni"}; + case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32: + case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32: + return {"+v"}; case MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16: case MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16: return {"+v", "+zvfh"}; @@ -617,6 +620,8 @@ } if (isRISCV64(config)) { static const MMAIntrinsic kAllRiscvV[] = { + MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F32_F32, + MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F32_F32, MMAIntrinsic::MMA_RISCV_V_VFMACC_1x8VLsx1_F16_F16, MMAIntrinsic::MMA_RISCV_V_VFMACC_8VLsx1x1_F16_F16, };
diff --git a/tests/e2e/matmul/BUILD.bazel b/tests/e2e/matmul/BUILD.bazel index 302a20f..749402c 100644 --- a/tests/e2e/matmul/BUILD.bazel +++ b/tests/e2e/matmul/BUILD.bazel
@@ -231,10 +231,13 @@ "--acc_type=%s" % acc_type, ] + (["--shapes=small"] if acc_type == "f64" else []), tags = ([ - # f16/bf16 trigger internal LLVM assertion errors on riscv and wasm. + # Large inner_tiled shapes time out under RISC-V QEMU (#24593). + # f16/bf16 also trigger LLVM assertion errors on riscv and wasm. "noriscv", "nowasm", - ] if (lhs_rhs_type == "f16" or lhs_rhs_type == "bf16") else []), + ] if (lhs_rhs_type == "f16" or lhs_rhs_type == "bf16") else [ + "noriscv", + ] if lhs_rhs_type == "f32" and acc_type == "f32" else []), target_backends_and_drivers = [ ("llvm-cpu", "local-task"), ], @@ -271,6 +274,28 @@ ("bf16", "f32"), ]] +# RISC-V only: small-shape coverage of the inner_tiled MMA path. The large +# default inner_tiled f32 set is tagged noriscv (#24593). +iree_generated_e2e_runner_test( + name = "e2e_matmul_cpu_dt_inner_tiled_riscv_small_f32_f32", + compiler_flags = [ + "--iree-opt-data-tiling", + "--iree-llvmcpu-enable-inner-tiled", + ], + generator = ":generate_e2e_matmul_tests", + generator_args = [ + "--lhs_rhs_type=f32", + "--acc_type=f32", + "--shapes=small", + ], + target_backends_and_drivers = [ + ("llvm-cpu", "local-task"), + ], + target_cpu_features_variants = ["riscv_64:rvv:+m,+v,+zvl512b"], + test_runner = "//tools/testing/e2e:iree-e2e-matmul-test", + test_type = "matmul", +) + # LLVMCPU, data-tiling lowering to iree_codegen.inner_tiled AND routing the # inner_tiled op through a built-in C-bitcode microkernel via # `--iree-llvmcpu-enable-llvm-ukernels=inner_tiled`. Restricted to the
diff --git a/tests/e2e/matmul/CMakeLists.txt b/tests/e2e/matmul/CMakeLists.txt index 70eb424..e790013 100644 --- a/tests/e2e/matmul/CMakeLists.txt +++ b/tests/e2e/matmul/CMakeLists.txt
@@ -566,7 +566,7 @@ "--iree-opt-data-tiling" "--iree-llvmcpu-enable-inner-tiled" LABELS - + "noriscv" TIMEOUT 300 TARGET_CPU_FEATURES_VARIANTS @@ -718,6 +718,30 @@ iree_generated_e2e_runner_test( NAME + e2e_matmul_cpu_dt_inner_tiled_riscv_small_f32_f32 + TEST_TYPE + matmul + GENERATOR + "generate_e2e_matmul_tests.py" + GENERATOR_ARGS + "--lhs_rhs_type=f32" + "--acc_type=f32" + "--shapes=small" + TEST_RUNNER + iree_tools_testing_e2e_iree-e2e-matmul-test + TARGET_BACKENDS + "llvm-cpu" + DRIVERS + "local-task" + COMPILER_FLAGS + "--iree-opt-data-tiling" + "--iree-llvmcpu-enable-inner-tiled" + TARGET_CPU_FEATURES_VARIANTS + "riscv_64:rvv:+m,+v,+zvl512b" +) + +iree_generated_e2e_runner_test( + NAME e2e_matmul_cpu_dt_inner_tiled_llvm_ukernel_bf16_f32 TEST_TYPE matmul