Add bufferization support for UKernel ops. (#12175)

Co-authored-by: bjacob <jacob.benoit.1@gmail.com>

This adds an ExternalModel implementation for the BufferizationInterface that derives from
bufferization::DstBufferizableOpInterfaceExternalModel, allowing the OneShot Bufferization to pick up these ops.
diff --git a/compiler/src/iree/compiler/Codegen/Common/test/iree_comprehensive_bufferize.mlir b/compiler/src/iree/compiler/Codegen/Common/test/iree_comprehensive_bufferize.mlir
index e578c1d..55db65b 100644
--- a/compiler/src/iree/compiler/Codegen/Common/test/iree_comprehensive_bufferize.mlir
+++ b/compiler/src/iree/compiler/Codegen/Common/test/iree_comprehensive_bufferize.mlir
@@ -2525,3 +2525,44 @@
 //       CHECK: hal.interface.binding.subspan set(0) binding(1) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>>
 //       CHECK: hal.interface.binding.subspan set(0) binding(2) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>>
 //       CHECK: hal.interface.binding.subspan set(0) binding(3) type(storage_buffer) : memref<?x?xf32, #hal.descriptor_type<storage_buffer>>
+
+// -----
+
+func.func @micro_kernel_op() {
+  %d0 = hal.interface.constant.load[0] : index
+  %d1 = hal.interface.constant.load[1] : index
+  %s0 = hal.interface.constant.load[2] : f32
+  %s1 = hal.interface.constant.load[3] : i64
+  %arg0_binding = hal.interface.binding.subspan set(0) binding(0) type(uniform_buffer) : !flow.dispatch.tensor<readonly:tensor<?x?xf32>>{%d0, %d1}
+  %arg1_binding = hal.interface.binding.subspan set(0) binding(1) type(uniform_buffer) : !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1}
+  %arg2_binding = hal.interface.binding.subspan set(0) binding(2) type(uniform_buffer) : !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1}
+  %arg3_binding = hal.interface.binding.subspan set(0) binding(3) type(uniform_buffer) : !flow.dispatch.tensor<readonly:tensor<?x?xf32>>{%d0, %d1}
+  %arg0 = flow.dispatch.tensor.load %arg0_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1]
+      : !flow.dispatch.tensor<readonly:tensor<?x?xf32>>{%d0, %d1} -> tensor<?x?xf32>
+  %arg1 = flow.dispatch.tensor.load %arg1_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1]
+      : !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1} -> tensor<?x?xf32>
+  %arg2 = flow.dispatch.tensor.load %arg2_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1]
+      : !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1} -> tensor<?x?xf32>
+  %arg3 = flow.dispatch.tensor.load %arg3_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1]
+      : !flow.dispatch.tensor<readonly:tensor<?x?xf32>>{%d0, %d1} -> tensor<?x?xf32>
+  %0:2 = iree_codegen.ukernel.generic "foo" ins(%arg0 : tensor<?x?xf32>)
+      outs(%arg1, %arg2 : tensor<?x?xf32>, tensor<?x?xf32>)
+      (%s0, %arg3, %s1 : f32, tensor<?x?xf32>, i64) -> tensor<?x?xf32>, tensor<?x?xf32>
+  flow.dispatch.tensor.store %0#0, %arg1_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1]
+      : tensor<?x?xf32> -> !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1}
+  flow.dispatch.tensor.store %0#1, %arg2_binding, offsets = [0, 0], sizes = [%d0, %d1], strides = [1, 1]
+      : tensor<?x?xf32> -> !flow.dispatch.tensor<writeonly:tensor<?x?xf32>>{%d0, %d1}
+  return
+}
+// CHECK-LABEL: func @micro_kernel_op()
+//   CHECK-DAG:   %[[S0:.+]] = hal.interface.constant.load[2]
+//   CHECK-DAG:   %[[S1:.+]] = hal.interface.constant.load[3]
+//   CHECK-DAG:   %[[ARG0:.+]] = hal.interface.binding.subspan set(0) binding(0) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>>
+//   CHECK-DAG:   %[[ARG1:.+]] = hal.interface.binding.subspan set(0) binding(1) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>>
+//   CHECK-DAG:   %[[ARG2:.+]] = hal.interface.binding.subspan set(0) binding(2) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>>
+//   CHECK-DAG:   %[[ARG3:.+]] = hal.interface.binding.subspan set(0) binding(3) type(uniform_buffer) : memref<?x?xf32, #hal.descriptor_type<uniform_buffer>>
+//       CHECK:   iree_codegen.ukernel.generic "foo"
+//  CHECK-SAME:       ins(%[[ARG0]] :
+//  CHECK-SAME:       outs(%[[ARG1]], %[[ARG2]] :
+//  CHECK-SAME:       (%[[S0]], %[[ARG3]], %[[S1]] :
+//       CHECK:   return
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/BUILD b/compiler/src/iree/compiler/Codegen/Dialect/BUILD
index a4acf2b..2b78e19 100644
--- a/compiler/src/iree/compiler/Codegen/Dialect/BUILD
+++ b/compiler/src/iree/compiler/Codegen/Dialect/BUILD
@@ -72,6 +72,7 @@
         "//runtime/src/iree/builtins/ukernel:exported_bits",
         "@llvm-project//llvm:Support",
         "@llvm-project//mlir:ArithDialect",
+        "@llvm-project//mlir:BufferizationDialect",
         "@llvm-project//mlir:DestinationStyleOpInterface",
         "@llvm-project//mlir:DialectUtils",
         "@llvm-project//mlir:FuncDialect",
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Dialect/CMakeLists.txt
index cfe30f3..70f2bdc 100644
--- a/compiler/src/iree/compiler/Codegen/Dialect/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/Dialect/CMakeLists.txt
@@ -36,6 +36,7 @@
     ::UKernelOpsGen
     LLVMSupport
     MLIRArithDialect
+    MLIRBufferizationDialect
     MLIRDestinationStyleOpInterface
     MLIRFuncDialect
     MLIRIR
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/IREECodegenDialect.h b/compiler/src/iree/compiler/Codegen/Dialect/IREECodegenDialect.h
index bdb9473..456b839 100644
--- a/compiler/src/iree/compiler/Codegen/Dialect/IREECodegenDialect.h
+++ b/compiler/src/iree/compiler/Codegen/Dialect/IREECodegenDialect.h
@@ -14,4 +14,12 @@
 #include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h.inc"  // IWYU pragma: keep
 // clang-format on
 
+namespace mlir {
+namespace iree_compiler {
+
+void registerUKernelBufferizationInterface(DialectRegistry &registry);
+
+}  // namespace iree_compiler
+}  // namespace mlir
+
 #endif  // IREE_COMPILER_CODEGEN_DIALECT_IREECODEGEN_DIALECT_H_
diff --git a/compiler/src/iree/compiler/Codegen/Dialect/UKernelOps.cpp b/compiler/src/iree/compiler/Codegen/Dialect/UKernelOps.cpp
index bb5b45a..33fd63f 100644
--- a/compiler/src/iree/compiler/Codegen/Dialect/UKernelOps.cpp
+++ b/compiler/src/iree/compiler/Codegen/Dialect/UKernelOps.cpp
@@ -7,10 +7,12 @@
 #include "iree/compiler/Codegen/Dialect/UKernelOps.h"
 
 #include "iree/builtins/ukernel/exported_bits.h"
+#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
 #include "iree/compiler/Codegen/Utils/EncodingInfo.h"
 #include "llvm/ADT/TypeSwitch.h"
 #include "llvm/Support/FormatVariadic.h"
 #include "mlir/Dialect/Arith/IR/Arith.h"
+#include "mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h"
 #include "mlir/Dialect/Func/IR/FuncOps.h"
 #include "mlir/Dialect/MemRef/IR/MemRef.h"
 #include "mlir/IR/OpImplementation.h"
@@ -327,8 +329,81 @@
       rewriter, getOperation(), fnName, fnArgTypes.value(),
       /*callReturnTypes=*/TypeRange{}, fnCallOperands.value());
 }
-
 }  // namespace Codegen
 }  // namespace IREE
+
+//===---------------------------------------------------------------------===//
+// Register bufferization interface.
+//===---------------------------------------------------------------------===//
+
+namespace {
+template <typename OpTy>
+struct UKernelOpsBufferizationInterface
+    : public bufferization::DstBufferizableOpInterfaceExternalModel<
+          UKernelOpsBufferizationInterface<OpTy>, OpTy> {
+  LogicalResult bufferize(
+      Operation *op, RewriterBase &rewriter,
+      const bufferization::BufferizationOptions &options) const {
+    // TODO: Handle operations with regions if needed.
+    if (op->getNumRegions() != 0) {
+      op->emitOpError(
+          "unhandled bufferization of micro kernel op with regions");
+    }
+    SmallVector<Value> bufferOpOperands;
+
+    // Replace all `tensor` operands with corresponding `memref` operands.
+    for (auto [index, operand] : llvm::enumerate(op->getOperands())) {
+      // For `tensor` type operands, replace with `memref` type operand.
+      if (operand.getType().template isa<RankedTensorType>()) {
+        FailureOr<Value> memrefOperand = getBuffer(rewriter, operand, options);
+        if (failed(memrefOperand)) {
+          return op->emitOpError(
+              llvm::formatv("failed to bufferize operand {0} ", index));
+        }
+        bufferOpOperands.push_back(memrefOperand.value());
+        continue;
+      }
+
+      // For all other operand types, just use the same value.
+      bufferOpOperands.push_back(operand);
+    }
+
+    // Ignore all result types that are tensor types.
+    SmallVector<Type> resultTypes;
+    for (auto resultType : op->getResultTypes()) {
+      if (resultType.isa<RankedTensorType>()) continue;
+      resultTypes.push_back(resultType);
+    }
+
+    auto bufferOp = rewriter.create<OpTy>(op->getLoc(), resultTypes,
+                                          bufferOpOperands, op->getAttrs());
+    SmallVector<Value> dpsInits =
+        cast<DestinationStyleOpInterface>(bufferOp.getOperation())
+            .getDpsInitOperands();
+    bufferization::replaceOpWithBufferizedValues(rewriter, op, dpsInits);
+    return success();
+  }
+};
+
+template <typename... Ops>
+struct RegisterUKernelOpsBufferizationInterface {
+  static void registerOpInterface(MLIRContext *context) {
+    (Ops::template attachInterface<UKernelOpsBufferizationInterface<Ops>>(
+         *context),
+     ...);
+  }
+};
+}  // namespace
+
+void registerUKernelBufferizationInterface(DialectRegistry &registry) {
+  registry.addExtension(
+      +[](MLIRContext *context, IREE::Codegen::IREECodegenDialect *dialect) {
+        RegisterUKernelOpsBufferizationInterface<
+#define GET_OP_LIST
+#include "iree/compiler/Codegen/Dialect/UKernelOps.cpp.inc"
+            >::registerOpInterface(context);
+      });
+}
+
 }  // namespace iree_compiler
 }  // namespace mlir
diff --git a/compiler/src/iree/compiler/Tools/init_iree_dialects.h b/compiler/src/iree/compiler/Tools/init_iree_dialects.h
index 5472749..c556c62 100644
--- a/compiler/src/iree/compiler/Tools/init_iree_dialects.h
+++ b/compiler/src/iree/compiler/Tools/init_iree_dialects.h
@@ -55,6 +55,7 @@
   // External models.
   IREE::Util::registerUtilExternalModels(registry);
   registerCodegenInterfaces(registry);
+  registerUKernelBufferizationInterface(registry);
 }
 
 }  // namespace iree_compiler