[LLVM integrate] drop carried LLVM commit reverts (#24779)
Adapt to LLVM bazel changes for config.h in musl, required by:
b8cc84591b6a [bazel] Correct config.h definitions for musl (#207295)
Remove per-function target attributes to re-eanble inlineing under
alwaysinline. Required by the combination of:
7bdca287b102 [InlineCost] Never inline functions with incompatible
target features (#205113)
37b8e765ce48 [AlwaysInliner] Do not inline on attribute mismatches
The changes have been assisted by Claude Code, but were manually checked
and cleaned.
---------
Signed-off-by: Stefan Schuermans <schuermans@roofline.ai>
Signed-off-by: Ege Beysel <beyselege@gmail.com>
Co-authored-by: Federico Bruzzone <federico.bruzzone.i@gmail.com>
Co-authored-by: Ege Beysel <beyselege@gmail.com>diff --git a/MODULE.bazel b/MODULE.bazel
index a75ab70..635390c 100644
--- a/MODULE.bazel
+++ b/MODULE.bazel
@@ -48,6 +48,12 @@
path = "third_party/llvm-project/utils/bazel",
)
+# llvm-project-overlay/llvm/config.bzl refers to @llvm//platforms/config:{gnu,musl}.
+# We need to teach the IREE bazel build about `@llvm`, so it can understand this
+# reference. The overlay declares @llvm as a dev_dependency, and we should mirror
+# it with the same dependency version here.
+bazel_dep(name = "llvm", version = "0.8.5")
+
# Third-party dependencies that LLVM's bzlmod setup now sources from the Bazel
# Central Registry as bazel_deps (previously generated by llvm_repos_extension).
# IREE only references nanobind directly (//runtime/bindings/python); gmp, mpc,
diff --git a/compiler/plugins/target/LLVMCPU/builtins/BUILD.bazel b/compiler/plugins/target/LLVMCPU/builtins/BUILD.bazel
index 651ca3e..cdedc6a 100644
--- a/compiler/plugins/target/LLVMCPU/builtins/BUILD.bazel
+++ b/compiler/plugins/target/LLVMCPU/builtins/BUILD.bazel
@@ -27,6 +27,7 @@
deps = [
"//compiler/src/iree/compiler/Codegen/Utils",
"//compiler/src/iree/compiler/Dialect/HAL/IR",
+ "//compiler/src/iree/compiler/Dialect/HAL/Utils:LLVMLinkerUtils",
"//runtime/src/iree/builtins/device:libdevice_bitcode",
"//runtime/src/iree/builtins/musl/bin:libmusl",
"//runtime/src/iree/builtins/ukernel:embed_ukernel_bitcode",
diff --git a/compiler/plugins/target/LLVMCPU/builtins/CMakeLists.txt b/compiler/plugins/target/LLVMCPU/builtins/CMakeLists.txt
index 5737b9b..885b28d 100644
--- a/compiler/plugins/target/LLVMCPU/builtins/CMakeLists.txt
+++ b/compiler/plugins/target/LLVMCPU/builtins/CMakeLists.txt
@@ -32,6 +32,7 @@
iree::builtins::ukernel::embed_ukernel_bitcode
iree::compiler::Codegen::Utils
iree::compiler::Dialect::HAL::IR
+ iree::compiler::Dialect::HAL::Utils::LLVMLinkerUtils
PUBLIC
)
diff --git a/compiler/plugins/target/LLVMCPU/builtins/ukernel/test/BUILD.bazel b/compiler/plugins/target/LLVMCPU/builtins/ukernel/test/BUILD.bazel
index a710d6a..5293261 100644
--- a/compiler/plugins/target/LLVMCPU/builtins/ukernel/test/BUILD.bazel
+++ b/compiler/plugins/target/LLVMCPU/builtins/ukernel/test/BUILD.bazel
@@ -27,6 +27,7 @@
),
cfg = "//compiler:lit.cfg.py",
tools = [
+ "//tools:iree-compile",
"//tools:iree-opt",
"@llvm-project//llvm:FileCheck",
],
diff --git a/compiler/plugins/target/LLVMCPU/builtins/ukernel/test/CMakeLists.txt b/compiler/plugins/target/LLVMCPU/builtins/ukernel/test/CMakeLists.txt
index 63f4c15..9373360 100644
--- a/compiler/plugins/target/LLVMCPU/builtins/ukernel/test/CMakeLists.txt
+++ b/compiler/plugins/target/LLVMCPU/builtins/ukernel/test/CMakeLists.txt
@@ -21,6 +21,7 @@
"select_ukernel.mlir"
TOOLS
FileCheck
+ iree-compile
iree-opt
)
diff --git a/compiler/plugins/target/ROCM/ROCMTargetUtils.cpp b/compiler/plugins/target/ROCM/ROCMTargetUtils.cpp
index e6bc477..f969365 100644
--- a/compiler/plugins/target/ROCM/ROCMTargetUtils.cpp
+++ b/compiler/plugins/target/ROCM/ROCMTargetUtils.cpp
@@ -69,6 +69,13 @@
// Ignore the data layout of the module we're importing. This avoids a
// warning from the linker.
bitcodeModule->setDataLayout(module->getDataLayout());
+ // Remove target-cpu attributes to ensure inlining works even under newer
+ // amdgpuX.YPY triples. Remove this once device libraries are updated to
+ // handle this better.
+ // Note: See also the "clang -mlink-buildin-bitcode" implementation
+ // addressing this issue by propagating function target attributes, more
+ // complex than just stripping like we do here.
+ stripFunctionTargetAttrs(*bitcodeModule);
if (linker.linkInModule(
std::move(bitcodeModule), llvm::Linker::Flags::LinkOnlyNeeded,
[](llvm::Module &M, const llvm::StringSet<> &GVS) {
@@ -89,6 +96,10 @@
llvm::LLVMContext &context) {
llvm::MemoryBufferRef bitcodeBufferRef(contents, filename);
auto setAlwaysInline = [&](llvm::Module &module) {
+ // Remove target-cpu attributes to ensure inlining works even under newer
+ // amdgpuX.YPY triples. Remove this once device libraries are updated to
+ // handle this better. See also note at the call of the same function above.
+ stripFunctionTargetAttrs(module);
for (auto &func : module.getFunctionList()) {
func.addFnAttr(llvm::Attribute::AlwaysInline);
}
diff --git a/compiler/plugins/target/ROCM/builtins/ukernel/test/CMakeLists.txt b/compiler/plugins/target/ROCM/builtins/ukernel/test/CMakeLists.txt
index 6434780..ab4a547 100644
--- a/compiler/plugins/target/ROCM/builtins/ukernel/test/CMakeLists.txt
+++ b/compiler/plugins/target/ROCM/builtins/ukernel/test/CMakeLists.txt
@@ -11,6 +11,7 @@
lit
SRCS
"argmax_linking.mlir"
+ "ukernel_inlining.mlir"
TOOLS
${IREE_LLD_TARGET}
FileCheck
diff --git a/compiler/plugins/target/ROCM/builtins/ukernel/test/ukernel_inlining.mlir b/compiler/plugins/target/ROCM/builtins/ukernel/test/ukernel_inlining.mlir
new file mode 100644
index 0000000..32040d2
--- /dev/null
+++ b/compiler/plugins/target/ROCM/builtins/ukernel/test/ukernel_inlining.mlir
@@ -0,0 +1,54 @@
+// RUN: rm -rf %t && mkdir -p %t
+// RUN: iree-compile --iree-hal-target-device=hip --iree-rocm-target=gfx942 \
+// RUN: --iree-rocm-enable-ukernels=all \
+// RUN: --iree-hal-dump-executable-intermediates-to=%t %s -o /dev/null
+// RUN: cat %t/*.linked.ll | FileCheck %s --check-prefix=LINKED
+// RUN: cat %t/*.optimized.ll | FileCheck %s --check-prefix=OPT
+
+// Regression test for the linked-in bitcode getting inlined into the kernel.
+//
+// Both the AMDGPU ukernels and the ROCm device library (ocml/ockl) are
+// compiled by clang, which leaves per-function "target-cpu"/"target-features"
+// attributes behind. The kernels IREE generates carry no such attributes, and
+// TargetTransformInfoImplBase::areInlineCompatible - which AMDGPU's TTI falls
+// through to - compares them as literal attribute values rather than as
+// resolved subtargets. So a callee that carries them is never considered
+// inline-compatible, even though it resolves to exactly the same subtarget via
+// the target machine, and even when marked `alwaysinline`.
+
+// First check that the ukernel and the device library are really being linked
+// in, so that this cannot silently pass by way of them never being selected.
+// LINKED: call {{.*}}@iree_uk_amdgpu_argmax_f32i64(
+// LINKED: call {{.*}}@__ockl_
+
+// Then check that neither survived as a call, i.e. both were inlined.
+// OPT: define {{.*}}@argmax_1d_f32i64_dispatch_0
+// OPT-NOT: call {{.*}}@iree_uk_amdgpu_
+// OPT-NOT: call {{.*}}@__ockl_
+
+func.func @argmax_1d_f32i64(%arg0: tensor<1x?xf32>) -> tensor<1x1xi64> {
+ %c0_i64 = arith.constant 0 : i64
+ %cst = arith.constant 0xFF800000 : f32
+ %0 = tensor.empty() : tensor<1xi64>
+ %1 = linalg.fill ins(%c0_i64 : i64) outs(%0 : tensor<1xi64>) -> tensor<1xi64>
+ %2 = tensor.empty() : tensor<1xf32>
+ %3 = linalg.fill ins(%cst : f32) outs(%2 : tensor<1xf32>) -> tensor<1xf32>
+ %4:2 = linalg.generic {
+ indexing_maps = [affine_map<(d0, d1) -> (d0, d1)>,
+ affine_map<(d0, d1) -> (d0)>,
+ affine_map<(d0, d1) -> (d0)>],
+ iterator_types = ["parallel", "reduction"]}
+ ins(%arg0 : tensor<1x?xf32>)
+ outs(%3, %1 : tensor<1xf32>, tensor<1xi64>) {
+ ^bb0(%in: f32, %out: f32, %out_0: i64):
+ %5 = linalg.index 1 : index
+ %6 = arith.index_cast %5 : index to i64
+ %7 = arith.maximumf %in, %out : f32
+ %8 = arith.cmpf ogt, %in, %out : f32
+ %9 = arith.select %8, %6, %out_0 : i64
+ linalg.yield %7, %9 : f32, i64
+ } -> (tensor<1xf32>, tensor<1xi64>)
+ %expanded = tensor.expand_shape %4#1 [[0, 1]] output_shape [1, 1]
+ : tensor<1xi64> into tensor<1x1xi64>
+ return %expanded : tensor<1x1xi64>
+}
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Utils/LLVMLinkerUtils.cpp b/compiler/src/iree/compiler/Dialect/HAL/Utils/LLVMLinkerUtils.cpp
index 5588cec..6c33598 100644
--- a/compiler/src/iree/compiler/Dialect/HAL/Utils/LLVMLinkerUtils.cpp
+++ b/compiler/src/iree/compiler/Dialect/HAL/Utils/LLVMLinkerUtils.cpp
@@ -22,6 +22,18 @@
"and is only linked if `arch` matches the target triple."),
llvm::cl::init(""));
+void stripFunctionTargetAttrs(llvm::Function &func) {
+ func.removeFnAttr("target-cpu");
+ func.removeFnAttr("tune-cpu");
+ func.removeFnAttr("target-features");
+}
+
+void stripFunctionTargetAttrs(llvm::Module &module) {
+ for (auto &func : module.functions()) {
+ stripFunctionTargetAttrs(func);
+ }
+}
+
bool anyRequiredSymbols(const llvm::Module &module, StringRef prefix) {
for (const auto &function : module.functions()) {
if (!function.isIntrinsic() && function.isDeclaration() &&
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Utils/LLVMLinkerUtils.h b/compiler/src/iree/compiler/Dialect/HAL/Utils/LLVMLinkerUtils.h
index 1ccbbf3..e3992e6 100644
--- a/compiler/src/iree/compiler/Dialect/HAL/Utils/LLVMLinkerUtils.h
+++ b/compiler/src/iree/compiler/Dialect/HAL/Utils/LLVMLinkerUtils.h
@@ -17,6 +17,13 @@
// Returns true if there are any external symbols in |module| with |prefix|.
bool anyRequiredSymbols(const llvm::Module &module, StringRef prefix);
+// Strips the per-function target attributes that clang embeds when compiling
+// a module. These make a function incompatible for inlining (even under
+// alwaysinline), so callers that force-inline bitcode need to strip those
+// first.
+void stripFunctionTargetAttrs(llvm::Function &func);
+void stripFunctionTargetAttrs(llvm::Module &module);
+
// User callback to inject custom global values or functions into |module| prior
// to linking.
using ModuleSpecializationCallback = std::function<void(llvm::Module &module)>;
diff --git a/third_party/llvm-project b/third_party/llvm-project
index e7fac4e..8c5193b 160000
--- a/third_party/llvm-project
+++ b/third_party/llvm-project
@@ -1 +1 @@
-Subproject commit e7fac4e39085e03399f11fd30f9e1fbe631071eb
+Subproject commit 8c5193b41f2f1f5fb704a70e1f4bb7c0153d2d9d
