Add bufferization support for UKernel ops. (#12175) Co-authored-by: bjacob <jacob.benoit.1@gmail.com> This adds an ExternalModel implementation for the BufferizationInterface that derives from bufferization::DstBufferizableOpInterfaceExternalModel, allowing the OneShot Bufferization to pick up these ops.
diff --git a/compiler/src/iree/compiler/Codegen/Common/test/iree_comprehensive_bufferize.mlir b/compiler/src/iree/compiler/Codegen/Common/test/iree_comprehensive_bufferize.mlir index e578c1d..55db65b 100644 --- a/compiler/src/iree/compiler/Codegen/Common/test/iree_comprehensive_bufferize.mlir +++ b/compiler/src/iree/compiler/Codegen/Common/test/iree_comprehensive_bufferize.mlir
@@ -2525,3 +2525,44 @@ // CHECK: hal.interface.binding.subspan set(0) binding(1) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>> // CHECK: hal.interface.binding.subspan set(0) binding(2) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>> // CHECK: hal.interface.binding.subspan set(0) binding(3) type(storage_buffer) : memref<?x?xf32, #hal.descriptor_type<storage_buffer>> + +// ----- + +func.func @micro_kernel_op() { + %d0 = hal.interface.constant.load[0] : index + %d1 = hal.interface.constant.load[1] : index + %s0 = hal.interface.constant.load[2] : f32 + %s1 = hal.interface.constant.load[3] : i64 + %arg0_binding = hal.interface.binding.subspan set(0) binding(0) type(uniform_buffer) : !flow.dispatch.tensor<readonly:tensor<?x?xf32>>{%d0, %d1} + %arg1_binding = hal.interface.binding.subspan set(0) binding(1) type(uniform_buffer) : !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1} + %arg2_binding = hal.interface.binding.subspan set(0) binding(2) type(uniform_buffer) : !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1} + %arg3_binding = hal.interface.binding.subspan set(0) binding(3) type(uniform_buffer) : !flow.dispatch.tensor<readonly:tensor<?x?xf32>>{%d0, %d1} + %arg0 = flow.dispatch.tensor.load %arg0_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1] + : !flow.dispatch.tensor<readonly:tensor<?x?xf32>>{%d0, %d1} -> tensor<?x?xf32> + %arg1 = flow.dispatch.tensor.load %arg1_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1] + : !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1} -> tensor<?x?xf32> + %arg2 = flow.dispatch.tensor.load %arg2_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1] + : !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1} -> tensor<?x?xf32> + %arg3 = flow.dispatch.tensor.load %arg3_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1] + : !flow.dispatch.tensor<readonly:tensor<?x?xf32>>{%d0, %d1} -> tensor<?x?xf32> + %0:2 = iree_codegen.ukernel.generic "foo" ins(%arg0 : tensor<?x?xf32>) + outs(%arg1, %arg2 : tensor<?x?xf32>, tensor<?x?xf32>) + (%s0, %arg3, %s1 : f32, tensor<?x?xf32>, i64) -> tensor<?x?xf32>, tensor<?x?xf32> + flow.dispatch.tensor.store %0#0, %arg1_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1] + : tensor<?x?xf32> -> !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1} + flow.dispatch.tensor.store %0#1, %arg2_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1] + : tensor<?x?xf32> -> !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1} + return +} +// CHECK-LABEL: func @micro_kernel_op() +// CHECK-DAG: %[[S0:.+]] = hal.interface.constant.load[2] +// CHECK-DAG: %[[S1:.+]] = hal.interface.constant.load[3] +// CHECK-DAG: %[[ARG0:.+]] = hal.interface.binding.subspan set(0) binding(0) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>> +// CHECK-DAG: %[[ARG1:.+]] = hal.interface.binding.subspan set(0) binding(1) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>> +// CHECK-DAG: %[[ARG2:.+]] = hal.interface.binding.subspan set(0) binding(2) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>> +// CHECK-DAG: %[[ARG3:.+]] = hal.interface.binding.subspan set(0) binding(3) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>> +// CHECK: iree_codegen.ukernel.generic "foo" +// CHECK-SAME: ins(%[[ARG0]] : +// CHECK-SAME: outs(%[[ARG1]], %[[ARG2]] : +// CHECK-SAME: (%[[S0]], %[[ARG3]], %[[S1]] : +// CHECK: return
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/BUILD b/compiler/src/iree/compiler/Codegen/Dialect/BUILD index a4acf2b..2b78e19 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/BUILD +++ b/compiler/src/iree/compiler/Codegen/Dialect/BUILD
@@ -72,6 +72,7 @@ "//runtime/src/iree/builtins/ukernel:exported_bits", "@llvm-project//llvm:Support", "@llvm-project//mlir:ArithDialect", + "@llvm-project//mlir:BufferizationDialect", "@llvm-project//mlir:DestinationStyleOpInterface", "@llvm-project//mlir:DialectUtils", "@llvm-project//mlir:FuncDialect",
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Dialect/CMakeLists.txt index cfe30f3..70f2bdc 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/CMakeLists.txt +++ b/compiler/src/iree/compiler/Codegen/Dialect/CMakeLists.txt
@@ -36,6 +36,7 @@ ::UKernelOpsGen LLVMSupport MLIRArithDialect + MLIRBufferizationDialect MLIRDestinationStyleOpInterface MLIRFuncDialect MLIRIR
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/IREECodegenDialect.h b/compiler/src/iree/compiler/Codegen/Dialect/IREECodegenDialect.h index bdb9473..456b839 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/IREECodegenDialect.h +++ b/compiler/src/iree/compiler/Codegen/Dialect/IREECodegenDialect.h
@@ -14,4 +14,12 @@ #include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h.inc" // IWYU pragma: keep // clang-format on +namespace mlir { +namespace iree_compiler { + +void registerUKernelBufferizationInterface(DialectRegistry ®istry); + +} // namespace iree_compiler +} // namespace mlir + #endif // IREE_COMPILER_CODEGEN_DIALECT_IREECODEGEN_DIALECT_H_
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/UKernelOps.cpp b/compiler/src/iree/compiler/Codegen/Dialect/UKernelOps.cpp index bb5b45a..33fd63f 100644 --- a/compiler/src/iree/compiler/Codegen/Dialect/UKernelOps.cpp +++ b/compiler/src/iree/compiler/Codegen/Dialect/UKernelOps.cpp
@@ -7,10 +7,12 @@ #include "iree/compiler/Codegen/Dialect/UKernelOps.h" #include "iree/builtins/ukernel/exported_bits.h" +#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h" #include "iree/compiler/Codegen/Utils/EncodingInfo.h" #include "llvm/ADT/TypeSwitch.h" #include "llvm/Support/FormatVariadic.h" #include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/MemRef/IR/MemRef.h" #include "mlir/IR/OpImplementation.h" @@ -327,8 +329,81 @@ rewriter, getOperation(), fnName, fnArgTypes.value(), /*callReturnTypes=*/TypeRange{}, fnCallOperands.value()); } - } // namespace Codegen } // namespace IREE + +//===---------------------------------------------------------------------===// +// Register bufferization interface. +//===---------------------------------------------------------------------===// + +namespace { +template <typename OpTy> +struct UKernelOpsBufferizationInterface + : public bufferization::DstBufferizableOpInterfaceExternalModel< + UKernelOpsBufferizationInterface<OpTy>, OpTy> { + LogicalResult bufferize( + Operation *op, RewriterBase &rewriter, + const bufferization::BufferizationOptions &options) const { + // TODO: Handle operations with regions if needed. + if (op->getNumRegions() != 0) { + op->emitOpError( + "unhandled bufferization of micro kernel op with regions"); + } + SmallVector<Value> bufferOpOperands; + + // Replace all `tensor` operands with corresponding `memref` operands. + for (auto [index, operand] : llvm::enumerate(op->getOperands())) { + // For `tensor` type operands, replace with `memref` type operand. + if (operand.getType().template isa<RankedTensorType>()) { + FailureOr<Value> memrefOperand = getBuffer(rewriter, operand, options); + if (failed(memrefOperand)) { + return op->emitOpError( + llvm::formatv("failed to bufferize operand {0} ", index)); + } + bufferOpOperands.push_back(memrefOperand.value()); + continue; + } + + // For all other operand types, just use the same value. + bufferOpOperands.push_back(operand); + } + + // Ignore all result types that are tensor types. + SmallVector<Type> resultTypes; + for (auto resultType : op->getResultTypes()) { + if (resultType.isa<RankedTensorType>()) continue; + resultTypes.push_back(resultType); + } + + auto bufferOp = rewriter.create<OpTy>(op->getLoc(), resultTypes, + bufferOpOperands, op->getAttrs()); + SmallVector<Value> dpsInits = + cast<DestinationStyleOpInterface>(bufferOp.getOperation()) + .getDpsInitOperands(); + bufferization::replaceOpWithBufferizedValues(rewriter, op, dpsInits); + return success(); + } +}; + +template <typename... Ops> +struct RegisterUKernelOpsBufferizationInterface { + static void registerOpInterface(MLIRContext *context) { + (Ops::template attachInterface<UKernelOpsBufferizationInterface<Ops>>( + *context), + ...); + } +}; +} // namespace + +void registerUKernelBufferizationInterface(DialectRegistry ®istry) { + registry.addExtension( + +[](MLIRContext *context, IREE::Codegen::IREECodegenDialect *dialect) { + RegisterUKernelOpsBufferizationInterface< +#define GET_OP_LIST +#include "iree/compiler/Codegen/Dialect/UKernelOps.cpp.inc" + >::registerOpInterface(context); + }); +} + } // namespace iree_compiler } // namespace mlir
diff --git a/compiler/src/iree/compiler/Tools/init_iree_dialects.h b/compiler/src/iree/compiler/Tools/init_iree_dialects.h index 5472749..c556c62 100644 --- a/compiler/src/iree/compiler/Tools/init_iree_dialects.h +++ b/compiler/src/iree/compiler/Tools/init_iree_dialects.h
@@ -55,6 +55,7 @@ // External models. IREE::Util::registerUtilExternalModels(registry); registerCodegenInterfaces(registry); + registerUKernelBufferizationInterface(registry); } } // namespace iree_compiler