[NFC-ish] Split out kernel config into <BACKEND>SelectLoweringStrategy (#15321)
This slightly restructures codegen in all of the backends such that the
selection of the lowering strategy is separate from the construction and
application of the relevant pipeline. This has two potential benefits:
1) By decoupling strategy selection from the lowering, we can move the
translation info onto function entry points (and/or individual private
functions) rather than the export op. This allows codegen to nest on the
function rather than the variant, whereas before this change, strategy
selection would still need the higher level of nesting.
2) We can now insert passes between strategy selection and application.
This allows expansion of variants based on the set of required
capabilities for each strategy, and contraction across variants that can
now be statically resolved as identical (think an elementwise operation
being independent of SubgroupNonUniform, now we can collapse two
variants where the presence of SubgroupNonUniform was the only thing
present).
Unfortunately for the time being, we still need to register essentially
the same set of dialects during the strategy selection pass because the
Transform Dialect builders in TransformStrategies can more or less
produce whatever they want. Once those strategies are deprecated we can
drop the extra registrations.
diff --git a/compiler/plugins/target/CUDA/CUDATarget.cpp b/compiler/plugins/target/CUDA/CUDATarget.cpp
index 6054688..31331e8 100644
--- a/compiler/plugins/target/CUDA/CUDATarget.cpp
+++ b/compiler/plugins/target/CUDA/CUDATarget.cpp
@@ -408,7 +408,7 @@
if (variantOp.isExternal())
return;
- buildLLVMGPUTransformPassPipeline(passManager, false);
+ buildLLVMGPUCodegenPassPipeline(passManager, false);
}
LogicalResult serializeExecutable(const SerializationOptions &serOptions,
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel
index 7b1ab5f..e5f36af 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel
@@ -60,6 +60,7 @@
"LLVMCPULowerToUKernels.cpp",
"LLVMCPUMmt4dVectorLowering.cpp",
"LLVMCPUPeel.cpp",
+ "LLVMCPUSelectLoweringStrategy.cpp",
"LLVMCPUSplitReduction.cpp",
"LLVMCPUSynchronizeSymbolVisibility.cpp",
"LLVMCPUTensorPad.cpp",
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt
index d798720..53ac5f4 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt
@@ -61,6 +61,7 @@
"LLVMCPULowerToUKernels.cpp"
"LLVMCPUMmt4dVectorLowering.cpp"
"LLVMCPUPeel.cpp"
+ "LLVMCPUSelectLoweringStrategy.cpp"
"LLVMCPUSplitReduction.cpp"
"LLVMCPUSynchronizeSymbolVisibility.cpp"
"LLVMCPUTensorPad.cpp"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp
index 5007f9a..f731f8f 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp
@@ -45,8 +45,7 @@
const LLVMCPULowerExecutableTargetPass &pass) {}
void getDependentDialects(DialectRegistry ®istry) const override {
// clang-format off
- registry.insert<IREE::Codegen::IREECodegenDialect,
- IREE::HAL::HALDialect,
+ registry.insert<IREE::HAL::HALDialect,
IREE::LinalgExt::IREELinalgExtDialect,
bufferization::BufferizationDialect,
linalg::LinalgDialect,
@@ -63,15 +62,6 @@
void runOnOperation() override;
private:
- Option<bool> testLoweringConfiguration{
- *this, "test-lowering-configuration",
- llvm::cl::desc(
- "Flag used for lit-testing the default configuration set for root "
- "ops in hal.executable.variants. Defaults to false and is set to "
- "true "
- "for lit tests. Not for general usage"),
- llvm::cl::init(false)};
-
Option<std::string> useLoweringPipeline{
*this, "use-lowering-pipeline",
llvm::cl::desc(
@@ -172,18 +162,10 @@
return signalPassFailure();
}
} else {
- // Use default heuristics.
- if (failed(initCPULaunchConfig(moduleOp))) {
- return signalPassFailure();
- }
-
// There might be multiple entry points in the module. Currently, all of
- // them need to have the same translation info.
- // TODO(ravishankarm): This is strange that this is not enforced
- // structurally, but something to address later on. The main issue is how
- // to invoke separate dynamic pass pipelines on entry point functions, when
- // the passes might have module level changes. For now this restriction
- // is fine.
+ // them need to have the same translation info. This should already be
+ // verified when the strategies are set, but we still need to retrieve the
+ // correct translation info.
llvm::StringMap<IREE::HAL::ExecutableExportOp> exportOps =
getAllEntryPoints(moduleOp);
std::optional<IREE::Codegen::TranslationInfoAttr> translationInfo;
@@ -196,6 +178,7 @@
moduleOp.emitOpError(
"unhandled compilation of entry point functions with different "
"translation info");
+ return signalPassFailure();
}
} else {
translationInfo = currTranslationInfo;
@@ -203,30 +186,7 @@
}
}
- // Verify the configuration.
if (translationInfo.has_value()) {
- LogicalResult verificationStatus = success();
- switch (translationInfo.value().getDispatchLoweringPassPipeline()) {
- case IREE::Codegen::DispatchLoweringPassPipeline::CPUDoubleTilingExpert:
- case IREE::Codegen::DispatchLoweringPassPipeline::
- CPUDoubleTilingPadExpert:
- verificationStatus = verifyLoweringConfiguration(
- moduleOp, translationInfo.value(),
- verifyDoubleTilingExpertPassPipelineConfig);
- break;
- case IREE::Codegen::DispatchLoweringPassPipeline::
- CPUConvTileAndDecomposeExpert:
- verificationStatus =
- verifyLoweringConfiguration(moduleOp, translationInfo.value(),
- verifyConvTileAndDecomposeExpertConfig);
- break;
- default:
- break;
- }
- if (failed(verificationStatus)) {
- return signalPassFailure();
- }
-
auto target = variantOp.getTarget();
bool lowerToAVX2 = hasAVX2Feature(target);
auto walkRes = moduleOp.walk([](linalg::LinalgOp linalgOp) {
@@ -242,79 +202,75 @@
bool enableMicrokernels = hasMicrokernels(target);
bool enableAArch64SSVE = isAArch64(target) && hasAnySVEFeature(target) &&
hasSMEFeature(target);
- if (!testLoweringConfiguration) {
- switch (translationInfo.value().getDispatchLoweringPassPipeline()) {
- // No pipleline specified, nothing to do.
- case IREE::Codegen::DispatchLoweringPassPipeline::None:
- return;
- case IREE::Codegen::DispatchLoweringPassPipeline::CPUDefault:
- addCPUDefaultPassPipeline(executableLoweringPipeline);
- break;
- case IREE::Codegen::DispatchLoweringPassPipeline::
- CPUBufferOpsTileAndVectorize: {
- TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
- addCPUBufferOpsTileAndVectorizePipeline(
- executableLoweringPipeline, tilingConfig, enableVectorMasking,
- enableAArch64SSVE);
- break;
- }
- case IREE::Codegen::DispatchLoweringPassPipeline::
- CPUDoubleTilingExpert: {
- TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
- addMultiTilingExpertPassPipeline(
- executableLoweringPipeline, tilingConfig,
- /*enablePeeling=*/false, enableVectorMasking, lowerToAVX2);
- break;
- }
- case IREE::Codegen::DispatchLoweringPassPipeline::
- CPUDoubleTilingPadExpert: {
- TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
- addDoubleTilingPadExpertPassPipeline(
- executableLoweringPipeline, tilingConfig, enableVectorMasking);
- break;
- }
- case IREE::Codegen::DispatchLoweringPassPipeline::
- CPUDoubleTilingPeelingExpert: {
- TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
- addMultiTilingExpertPassPipeline(
- executableLoweringPipeline, tilingConfig,
- /*enablePeeling=*/true, enableVectorMasking, lowerToAVX2,
- enableAArch64SSVE);
- break;
- }
- case IREE::Codegen::DispatchLoweringPassPipeline::
- CPUConvTileAndDecomposeExpert: {
- TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
- addConvTileAndDecomposeExpertPassPipeline(
- executableLoweringPipeline, tilingConfig, enableVectorMasking,
- enableAArch64SSVE);
- break;
- }
- case IREE::Codegen::DispatchLoweringPassPipeline::Mmt4dTilingExpert: {
- TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
- addMmt4dTilingExpertPassPipeline(executableLoweringPipeline,
- tilingConfig, enableMicrokernels);
- break;
- }
- case IREE::Codegen::DispatchLoweringPassPipeline::CPUDataTiling: {
- TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
- addCPUDataTilingPipeline(executableLoweringPipeline, tilingConfig,
- enableVectorMasking);
- break;
- }
- case IREE::Codegen::DispatchLoweringPassPipeline::VMVXDefault:
- addVMVXDefaultPassPipeline(executableLoweringPipeline,
- enableMicrokernels);
- break;
- // Transform-dialect pipelines.
- case IREE::Codegen::DispatchLoweringPassPipeline::
- TransformDialectCodegen:
- addTransformDialectPasses(executableLoweringPipeline);
- break;
- default:
- moduleOp.emitOpError("Unsupported pipeline on CPU target.");
- return signalPassFailure();
- }
+ switch (translationInfo.value().getDispatchLoweringPassPipeline()) {
+ // No pipleline specified, nothing to do.
+ case IREE::Codegen::DispatchLoweringPassPipeline::None:
+ return;
+ case IREE::Codegen::DispatchLoweringPassPipeline::CPUDefault:
+ addCPUDefaultPassPipeline(executableLoweringPipeline);
+ break;
+ case IREE::Codegen::DispatchLoweringPassPipeline::
+ CPUBufferOpsTileAndVectorize: {
+ TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
+ addCPUBufferOpsTileAndVectorizePipeline(
+ executableLoweringPipeline, tilingConfig, enableVectorMasking,
+ enableAArch64SSVE);
+ break;
+ }
+ case IREE::Codegen::DispatchLoweringPassPipeline::CPUDoubleTilingExpert: {
+ TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
+ addMultiTilingExpertPassPipeline(
+ executableLoweringPipeline, tilingConfig,
+ /*enablePeeling=*/false, enableVectorMasking, lowerToAVX2);
+ break;
+ }
+ case IREE::Codegen::DispatchLoweringPassPipeline::
+ CPUDoubleTilingPadExpert: {
+ TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
+ addDoubleTilingPadExpertPassPipeline(executableLoweringPipeline,
+ tilingConfig, enableVectorMasking);
+ break;
+ }
+ case IREE::Codegen::DispatchLoweringPassPipeline::
+ CPUDoubleTilingPeelingExpert: {
+ TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
+ addMultiTilingExpertPassPipeline(
+ executableLoweringPipeline, tilingConfig,
+ /*enablePeeling=*/true, enableVectorMasking, lowerToAVX2,
+ enableAArch64SSVE);
+ break;
+ }
+ case IREE::Codegen::DispatchLoweringPassPipeline::
+ CPUConvTileAndDecomposeExpert: {
+ TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
+ addConvTileAndDecomposeExpertPassPipeline(
+ executableLoweringPipeline, tilingConfig, enableVectorMasking,
+ enableAArch64SSVE);
+ break;
+ }
+ case IREE::Codegen::DispatchLoweringPassPipeline::Mmt4dTilingExpert: {
+ TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
+ addMmt4dTilingExpertPassPipeline(executableLoweringPipeline,
+ tilingConfig, enableMicrokernels);
+ break;
+ }
+ case IREE::Codegen::DispatchLoweringPassPipeline::CPUDataTiling: {
+ TilingConfig tilingConfig = getTilingConfigForPipeline(moduleOp);
+ addCPUDataTilingPipeline(executableLoweringPipeline, tilingConfig,
+ enableVectorMasking);
+ break;
+ }
+ case IREE::Codegen::DispatchLoweringPassPipeline::VMVXDefault:
+ addVMVXDefaultPassPipeline(executableLoweringPipeline,
+ enableMicrokernels);
+ break;
+ // Transform-dialect pipelines.
+ case IREE::Codegen::DispatchLoweringPassPipeline::TransformDialectCodegen:
+ addTransformDialectPasses(executableLoweringPipeline);
+ break;
+ default:
+ moduleOp.emitOpError("Unsupported pipeline on CPU target.");
+ return signalPassFailure();
}
}
}
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSelectLoweringStrategy.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSelectLoweringStrategy.cpp
new file mode 100644
index 0000000..8ed54bd
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSelectLoweringStrategy.cpp
@@ -0,0 +1,154 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
+#include "iree/compiler/Codegen/Common/TileSizeSelection.h"
+#include "iree/compiler/Codegen/Dialect/IREECodegenAttrs.h"
+#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
+#include "iree/compiler/Codegen/LLVMCPU/KernelDispatch.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
+#include "iree/compiler/Codegen/LLVMCPU/Utils.h"
+#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
+#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
+#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
+#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
+#include "mlir/Dialect/PDL/IR/PDL.h"
+#include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Pass/PassRegistry.h"
+
+using mlir::iree_compiler::IREE::Codegen::LoweringConfigAttr;
+
+namespace mlir {
+namespace iree_compiler {
+
+namespace {
+/// Selects the lowering strategy for a hal.executable.variant operation.
+class LLVMCPUSelectLoweringStrategyPass
+ : public LLVMCPUSelectLoweringStrategyBase<
+ LLVMCPUSelectLoweringStrategyPass> {
+public:
+ LLVMCPUSelectLoweringStrategyPass() = default;
+ LLVMCPUSelectLoweringStrategyPass(
+ const LLVMCPUSelectLoweringStrategyPass &pass) {}
+ void getDependentDialects(DialectRegistry ®istry) const override {
+ // TODO(qedawkins): Once TransformStrategies is deprecated, drop the
+ // unnecessary dialect registrations.
+ // clang-format off
+ registry.insert<IREE::Codegen::IREECodegenDialect,
+ IREE::HAL::HALDialect,
+ IREE::LinalgExt::IREELinalgExtDialect,
+ bufferization::BufferizationDialect,
+ linalg::LinalgDialect,
+ LLVM::LLVMDialect,
+ pdl::PDLDialect,
+ pdl_interp::PDLInterpDialect,
+ scf::SCFDialect,
+ tensor::TensorDialect,
+ transform::TransformDialect,
+ vector::VectorDialect>();
+ // clang-format on
+ }
+
+ void runOnOperation() override;
+};
+} // namespace
+
+/// Verify that valid configuration is set for all ops within the module.
+template <typename F>
+static LogicalResult
+verifyLoweringConfiguration(ModuleOp module,
+ IREE::Codegen::TranslationInfoAttr translationInfo,
+ F verificationFn) {
+ auto walkResult = module.walk([&](Operation *op) -> WalkResult {
+ IREE::Codegen::LoweringConfigAttr loweringConfig = getLoweringConfig(op);
+ if (!loweringConfig)
+ return WalkResult::advance();
+ TilingConfig tilingConfig(loweringConfig);
+ return verificationFn(op, tilingConfig, translationInfo,
+ ArrayRef<int64_t>{});
+ });
+ return failure(walkResult.wasInterrupted());
+}
+
+void LLVMCPUSelectLoweringStrategyPass::runOnOperation() {
+ IREE::HAL::ExecutableVariantOp variantOp = getOperation();
+ ModuleOp moduleOp = variantOp.getInnerModule();
+ if (!moduleOp) {
+ getOperation()->emitError(
+ "Expected a variantOp root with an inner ModuleOp");
+ return signalPassFailure();
+ }
+
+ // Set the strategy with default heuristics.
+ if (failed(initCPULaunchConfig(moduleOp))) {
+ return signalPassFailure();
+ }
+
+ // There might be multiple entry points in the module. Currently, all of
+ // them need to have the same translation info.
+ // TODO(ravishankarm): This is strange that this is not enforced
+ // structurally, but something to address later on. The main issue is how
+ // to invoke separate dynamic pass pipelines on entry point functions, when
+ // the passes might have module level changes. For now this restriction
+ // is fine.
+ llvm::StringMap<IREE::HAL::ExecutableExportOp> exportOps =
+ getAllEntryPoints(moduleOp);
+ std::optional<IREE::Codegen::TranslationInfoAttr> translationInfo;
+ for (auto &it : exportOps) {
+ auto exportOp = it.second;
+ if (IREE::Codegen::TranslationInfoAttr currTranslationInfo =
+ getTranslationInfo(exportOp)) {
+ if (translationInfo) {
+ if (currTranslationInfo != translationInfo.value()) {
+ moduleOp.emitOpError(
+ "unhandled compilation of entry point functions with different "
+ "translation info");
+ return signalPassFailure();
+ }
+ } else {
+ translationInfo = currTranslationInfo;
+ }
+ }
+ }
+
+ // Verify the configuration.
+ if (translationInfo.has_value()) {
+ LogicalResult verificationStatus = success();
+ switch (translationInfo.value().getDispatchLoweringPassPipeline()) {
+ case IREE::Codegen::DispatchLoweringPassPipeline::CPUDoubleTilingExpert:
+ case IREE::Codegen::DispatchLoweringPassPipeline::CPUDoubleTilingPadExpert:
+ verificationStatus = verifyLoweringConfiguration(
+ moduleOp, translationInfo.value(),
+ verifyDoubleTilingExpertPassPipelineConfig);
+ break;
+ case IREE::Codegen::DispatchLoweringPassPipeline::
+ CPUConvTileAndDecomposeExpert:
+ verificationStatus =
+ verifyLoweringConfiguration(moduleOp, translationInfo.value(),
+ verifyConvTileAndDecomposeExpertConfig);
+ break;
+ default:
+ break;
+ }
+ if (failed(verificationStatus)) {
+ return signalPassFailure();
+ }
+ }
+}
+
+std::unique_ptr<OperationPass<IREE::HAL::ExecutableVariantOp>>
+createLLVMCPUSelectLoweringStrategyPass() {
+ return std::make_unique<LLVMCPUSelectLoweringStrategyPass>();
+}
+
+} // namespace iree_compiler
+} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp
index 2efcd8d..900d564 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp
@@ -751,7 +751,8 @@
passManager.addNestedPass<LLVM::LLVMFuncOp>(createAddFastMathFlagsPass());
}
-void buildLLVMCPUCodegenPassPipeline(OpPassManager &passManager) {
+void buildLLVMCPUCodegenStrategyInitializationPassPipeline(
+ OpPassManager &passManager) {
{
addCommonTargetExecutablePreprocessingPasses(passManager);
OpPassManager &modulePassManager = passManager.nest<ModuleOp>();
@@ -768,6 +769,11 @@
modulePassManager.addPass(createEraseHALDescriptorTypeFromMemRefPass());
}
+ passManager.addPass(createLLVMCPUSelectLoweringStrategyPass());
+}
+
+void buildLLVMCPUCodegenPassPipeline(OpPassManager &passManager) {
+ buildLLVMCPUCodegenStrategyInitializationPassPipeline(passManager);
passManager.addPass(createLLVMCPULowerExecutableTargetPass());
OpPassManager &nestedModulePM = passManager.nest<ModuleOp>();
addLowerToLLVMPasses(nestedModulePM);
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h
index 47dad29..c403fd4 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h
@@ -31,6 +31,12 @@
std::unique_ptr<OperationPass<func::FuncOp>>
createLLVMCPUEmitVectorizationRemarksPass();
+/// Pass to select a lowering strategy for a hal.executable.variant operation.
+/// The variant is annotated with the selected strategies, which are
+/// subsequently ingested by LLVMCPULowerExecutableTargetPass.
+std::unique_ptr<OperationPass<IREE::HAL::ExecutableVariantOp>>
+createLLVMCPUSelectLoweringStrategyPass();
+
/// Pass to lower the module an hal.executable.variant operation to external
/// dialect. Currently this pass lowers to LLVM dialect, but could be
/// generalized to lower to any "final" dialect like SPIR-V/NVVM, etc.
@@ -183,6 +189,10 @@
// LLVMCPU Pass Pipelines for lowering to LLVM dialect.
//----------------------------------------------------------------------------//
+/// Populates passes needed for preprocessing before codegen lowerings, as well
+/// as high level lowering strategy selection.
+void buildLLVMCPUCodegenRoutingPassPipeline(OpPassManager &passManager);
+
/// Populates passes needed to lower a XLA HLO op to LLVM dialect via the
/// structured ops path. The pass manager `pm` in here should operate on the
/// module within the IREE::HAL::ExecutableOp.
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td
index 69fc43f..eb5b894 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td
@@ -100,6 +100,15 @@
"mlir::iree_compiler::createLLVMCPUPeelPass()";
}
+def LLVMCPUSelectLoweringStrategy :
+ Pass<"iree-llvmcpu-select-lowering-strategy",
+ "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
+ let summary =
+ "Select a IREE::HAL::DispatchLoweringPassPipeline for lowering the variant";
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPUSelectLoweringStrategyPass()";
+}
+
def LLVMCPUSplitReduction : Pass<"iree-llvmcpu-split-reduction", "func::FuncOp"> {
let summary = "Pass to splitReduce linalg operations.";
let constructor = "mlir::iree_compiler::createLLVMCPUSplitReductionPass()";
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/data_tiling_pipeline.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/data_tiling_pipeline.mlir
index e7675ec..e79f3e0 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/data_tiling_pipeline.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/data_tiling_pipeline.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target)))' --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' --split-input-file %s | FileCheck %s
hal.executable private @aligned_generic_pack {
hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {cpu_features = "+avx512f", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128", native_vector_size = 64 : index, target_triple = "x86_64-none-elf"}>) {
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/illegal_configuration.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/illegal_configuration.mlir
index 70237d8..ad90f6c 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/illegal_configuration.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/illegal_configuration.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target{test-lowering-configuration=true})))' --verify-diagnostics --split-input-file %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy)))' --verify-diagnostics --split-input-file %s
#config = #iree_codegen.lowering_config<tile_sizes = []>
#translation = #iree_codegen.translation_info<CPUDoubleTilingExpert>
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_aarch64_launch_configuration.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_aarch64_launch_configuration.mlir
index ff4cff5..f86d347 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_aarch64_launch_configuration.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_aarch64_launch_configuration.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target{test-lowering-configuration=true})))' --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy)))' --split-input-file %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_configuration_without_distribution.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_configuration_without_distribution.mlir
index c87a942..fe65482 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_configuration_without_distribution.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_configuration_without_distribution.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target{test-lowering-configuration=true})))' --iree-codegen-llvm-disable-distribution --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy)))' --iree-codegen-llvm-disable-distribution --split-input-file %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_riscv_launch_configuration.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_riscv_launch_configuration.mlir
index aae2c8c..ce87f71 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_riscv_launch_configuration.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_riscv_launch_configuration.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target{test-lowering-configuration=true})))' --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy)))' --split-input-file %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_vmvx_launch_configuration.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_vmvx_launch_configuration.mlir
index 4517b55..8c1032b 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_vmvx_launch_configuration.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_vmvx_launch_configuration.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt -pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target{test-lowering-configuration=true})))' -split-input-file %s | FileCheck %s
+// RUN: iree-opt -pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy)))' -split-input-file %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_x86_64_launch_configuration.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_x86_64_launch_configuration.mlir
index 18694fb..b2b3a4a 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_x86_64_launch_configuration.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/materialize_x86_64_launch_configuration.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmcpu-lower-executable-target{test-lowering-configuration=true})))' --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmcpu-select-lowering-strategy)))' --split-input-file %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pad_conv_pipeline_tests.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pad_conv_pipeline_tests.mlir
index 3a49ce9..b69ed56 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pad_conv_pipeline_tests.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pad_conv_pipeline_tests.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target)))' --iree-llvmcpu-enable-pad-consumer-fusion --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' --iree-llvmcpu-enable-pad-consumer-fusion --split-input-file %s | FileCheck %s
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu = "generic", cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128", native_vector_size = 32 : index, target_triple = "x86_64-none-elf"}>
#map = affine_map<(d0, d1, d2, d3) -> (d0, d1, d2, d3)>
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pad_pipeline_tests.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pad_pipeline_tests.mlir
index 973771f..c7c35e7 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pad_pipeline_tests.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pad_pipeline_tests.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target)))" --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))" --split-input-file %s | FileCheck %s
hal.executable private @pad_only {
hal.executable.variant public @embedded_elf_x86_64 target(<"llvm-cpu", "embedded-elf-x86_64", {
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/peel_and_vectorize.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/peel_and_vectorize.mlir
index d8bc363..decd671 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/peel_and_vectorize.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/peel_and_vectorize.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmcpu-lower-executable-target)))' -split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' -split-input-file %s | FileCheck %s
// Test peeling + vectorization using CPUDoubleTilingPeelingExpert.
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pipeline_tests.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pipeline_tests.mlir
index 8eeacdb..68b0932 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pipeline_tests.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/pipeline_tests.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmcpu-lower-executable-target)))' --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' --split-input-file %s | FileCheck %s
// Check that this dispatch compiles to vectors and that there are no allocas.
// By proxy checks that destination passing style kicked in correctly
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/split_reduction_pipeline_tests.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/split_reduction_pipeline_tests.mlir
index 85aa23c..317df95 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/split_reduction_pipeline_tests.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/split_reduction_pipeline_tests.mlir
@@ -1,5 +1,5 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target)))' --iree-llvmcpu-reassociate-fp-reductions=false --split-input-file %s | FileCheck %s
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target)))' --iree-llvmcpu-reassociate-fp-reductions=true --split-input-file %s | FileCheck %s --check-prefix=REORDERCHECK
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' --iree-llvmcpu-reassociate-fp-reductions=false --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' --iree-llvmcpu-reassociate-fp-reductions=true --split-input-file %s | FileCheck %s --check-prefix=REORDERCHECK
#executable_target_embedded_elf_x86_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-x86_64", {cpu_features = "", data_layout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128", native_vector_size = 16 : index, target_triple = "x86_64-none-elf"}>
#map0 = affine_map<(d0, d1, d2) -> (d0, d1, d2)>
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/test_config_mmt4d.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/test_config_mmt4d.mlir
index 6d55666..bb165b9 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/test_config_mmt4d.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/test_config_mmt4d.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy)))' %s | FileCheck %s
#executable_target_embedded_elf_arm_64_ = #hal.executable.target<"llvm-cpu", "embedded-elf-arm_64", {data_layout = "e-m:e-i8:8:32-i16:16:32-i64:64-i128:128-n32:64-S128", native_vector_size = 16 : index, target_triple = "aarch64-none-elf"}>
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/transpose_avx2_lowering.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/transpose_avx2_lowering.mlir
index e93570b..5ac00ba 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/transpose_avx2_lowering.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/transpose_avx2_lowering.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target{test-lowering-configuration=false})))' --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' --split-input-file %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/vector_masking.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/vector_masking.mlir
index 7e0d4da..fa164d4 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/vector_masking.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/vector_masking.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target)))' -split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' -split-input-file %s | FileCheck %s
#compilation = #iree_codegen.compilation_info<
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/vectorize_nd_extract.mlir b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/vectorize_nd_extract.mlir
index ccda83c..fc36a24 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/test/vectorize_nd_extract.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/test/vectorize_nd_extract.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target)))' --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' --split-input-file %s | FileCheck %s
module attributes {hal.device.targets = [#hal.device.target<"llvm-cpu", {executable_targets = [#hal.executable.target<"llvm-cpu", "system-elf-riscv_64", {cpu = "generic-rv64", cpu_features = "+m,+a,+f,+d,+v", data_layout = "e-m:e-p:64:64-i64:64-i128:128-n32:64-S128", native_vector_size = 64 : index, target_triple = "riscv64"}>]}>]} {
hal.executable private @main_dispatch_77 {
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/LLVMGPU/BUILD.bazel
index 0a1f3c2..f1746e4 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/BUILD.bazel
@@ -56,6 +56,7 @@
"LLVMGPUCastAddressSpaceFunction.cpp",
"LLVMGPULowerExecutableTarget.cpp",
"LLVMGPUPackSharedMemoryAlloc.cpp",
+ "LLVMGPUSelectLoweringStrategy.cpp",
"LLVMGPUTensorCoreVectorization.cpp",
"LLVMGPUTensorPad.cpp",
"LLVMGPUTileAndDistribute.cpp",
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/LLVMGPU/CMakeLists.txt
index 0b8aa2d..8aa547d 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/CMakeLists.txt
@@ -55,6 +55,7 @@
"LLVMGPUCastAddressSpaceFunction.cpp"
"LLVMGPULowerExecutableTarget.cpp"
"LLVMGPUPackSharedMemoryAlloc.cpp"
+ "LLVMGPUSelectLoweringStrategy.cpp"
"LLVMGPUTensorCoreVectorization.cpp"
"LLVMGPUTensorPad.cpp"
"LLVMGPUTileAndDistribute.cpp"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPULowerExecutableTarget.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPULowerExecutableTarget.cpp
index 3437524..c1562d3 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPULowerExecutableTarget.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPULowerExecutableTarget.cpp
@@ -42,8 +42,7 @@
void getDependentDialects(DialectRegistry ®istry) const override {
// clang-format off
registry
- .insert<IREE::Codegen::IREECodegenDialect,
- IREE::HAL::HALDialect,
+ .insert<IREE::HAL::HALDialect,
IREE::LinalgExt::IREELinalgExtDialect,
linalg::LinalgDialect,
gpu::GPUDialect,
@@ -62,68 +61,19 @@
const LLVMGPULowerExecutableTargetPass &pass){};
void runOnOperation() override;
-
-private:
- Option<bool> testLoweringConfiguration{
- *this, "test-lowering-configuration",
- llvm::cl::desc(
- "Flag used for lit-testing the default configuration set for root "
- "ops in hal.executable.variants. Defaults to false and is set to "
- "true "
- "for lit tests. Not for general usage"),
- llvm::cl::init(false)};
};
} // namespace
-/// Verify that valid configuration is set for all ops within the compiled
-/// module.
-template <typename F>
-static LogicalResult
-verifyLoweringConfiguration(ModuleOp module,
- IREE::Codegen::TranslationInfoAttr translationInfo,
- ArrayRef<int64_t> workgroupSize, F verificationFn) {
- auto walkResult = module.walk([&](Operation *op) -> WalkResult {
- IREE::Codegen::LoweringConfigAttr loweringConfig = getLoweringConfig(op);
- if (!loweringConfig)
- return WalkResult::advance();
- return verificationFn(op, loweringConfig, translationInfo, workgroupSize);
- });
- return failure(walkResult.wasInterrupted());
-}
-
-static LogicalResult
-verifyEntryPoint(ModuleOp moduleOp,
- IREE::Codegen::TranslationInfoAttr translationInfo,
- IREE::HAL::ExecutableExportOp exportOp) {
- std::optional<mlir::ArrayAttr> workgroupSizeAttr =
- exportOp.getWorkgroupSize();
-
- if (workgroupSizeAttr.has_value()) {
- std::array<int64_t, 3> workgroupSizes;
- for (auto [index, attr] : llvm::enumerate(workgroupSizeAttr.value())) {
- workgroupSizes[index] = llvm::cast<IntegerAttr>(attr).getInt();
- }
- return verifyLoweringConfiguration(moduleOp, translationInfo,
- workgroupSizes, verifyGPUMatmulPipeline);
- }
- return success();
-}
-
void LLVMGPULowerExecutableTargetPass::runOnOperation() {
IREE::HAL::ExecutableVariantOp variantOp = getOperation();
ModuleOp moduleOp = variantOp.getInnerModule();
OpPassManager executableLoweringPipeline(
IREE::HAL::ExecutableVariantOp::getOperationName());
- if (failed(initGPULaunchConfig(moduleOp))) {
- return signalPassFailure();
- }
-
// There might be multiple entry points in the module. Currently, all of
- // them need to have the same pipeline.
- // TODO(ravishankarm): This is strange that this is not enforced
- // structurally, but something to address later on. For now this restriction
- // is fine.
+ // them need to have the same pipeline. This should have been verified during
+ // strategy selection, but we still need to retrieve the translation info
+ // here.
llvm::StringMap<IREE::HAL::ExecutableExportOp> exportOps =
getAllEntryPoints(moduleOp);
std::optional<IREE::Codegen::TranslationInfoAttr> translationInfo;
@@ -140,16 +90,10 @@
} else {
translationInfo = currTranslationInfo;
}
-
- // Verify the properties of each entry point based on the target
- // pipeline.
- if (failed(verifyEntryPoint(moduleOp, currTranslationInfo, exportOp))) {
- return signalPassFailure();
- }
}
}
- if (!testLoweringConfiguration && translationInfo.has_value()) {
+ if (translationInfo.has_value()) {
switch (translationInfo.value().getDispatchLoweringPassPipeline()) {
case IREE::Codegen::DispatchLoweringPassPipeline::LLVMGPUDefault:
addGPUDefaultPassPipeline(executableLoweringPipeline);
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUSelectLoweringStrategy.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUSelectLoweringStrategy.cpp
new file mode 100644
index 0000000..a572a81
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUSelectLoweringStrategy.cpp
@@ -0,0 +1,145 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
+#include "iree/compiler/Codegen/Dialect/IREECodegenAttrs.h"
+#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
+#include "iree/compiler/Codegen/LLVMGPU/KernelConfig.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
+#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
+#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
+#include "mlir/Dialect/PDL/IR/PDL.h"
+#include "mlir/Dialect/PDLInterp/IR/PDLInterp.h"
+#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Pass/PassRegistry.h"
+#include "mlir/Transforms/Passes.h"
+
+namespace mlir {
+namespace iree_compiler {
+
+namespace {
+/// Selects a lowering strategy for taking a hal.executable.variant operation
+/// to scalar/native-vector code.
+class LLVMGPUSelectLoweringStrategyPass
+ : public LLVMGPUSelectLoweringStrategyBase<
+ LLVMGPUSelectLoweringStrategyPass> {
+public:
+ void getDependentDialects(DialectRegistry ®istry) const override {
+ // TODO(qedawkins): Once TransformStrategies is deprecated, drop the
+ // unnecessary dialect registrations.
+ // clang-format off
+ registry
+ .insert<IREE::Codegen::IREECodegenDialect,
+ IREE::HAL::HALDialect,
+ IREE::LinalgExt::IREELinalgExtDialect,
+ linalg::LinalgDialect,
+ gpu::GPUDialect,
+ nvgpu::NVGPUDialect,
+ pdl::PDLDialect,
+ pdl_interp::PDLInterpDialect,
+ scf::SCFDialect,
+ tensor::TensorDialect,
+ transform::TransformDialect,
+ vector::VectorDialect>();
+ // clang-format on
+ }
+
+ LLVMGPUSelectLoweringStrategyPass() = default;
+ LLVMGPUSelectLoweringStrategyPass(
+ const LLVMGPUSelectLoweringStrategyPass &pass){};
+
+ void runOnOperation() override;
+};
+} // namespace
+
+/// Verify that valid configuration is set for all ops within the compiled
+/// module.
+template <typename F>
+static LogicalResult
+verifyLoweringConfiguration(ModuleOp module,
+ IREE::Codegen::TranslationInfoAttr translationInfo,
+ ArrayRef<int64_t> workgroupSize, F verificationFn) {
+ auto walkResult = module.walk([&](Operation *op) -> WalkResult {
+ IREE::Codegen::LoweringConfigAttr loweringConfig = getLoweringConfig(op);
+ if (!loweringConfig)
+ return WalkResult::advance();
+ return verificationFn(op, loweringConfig, translationInfo, workgroupSize);
+ });
+ return failure(walkResult.wasInterrupted());
+}
+
+static LogicalResult
+verifyEntryPoint(ModuleOp moduleOp,
+ IREE::Codegen::TranslationInfoAttr translationInfo,
+ IREE::HAL::ExecutableExportOp exportOp) {
+ std::optional<mlir::ArrayAttr> workgroupSizeAttr =
+ exportOp.getWorkgroupSize();
+
+ if (workgroupSizeAttr.has_value()) {
+ std::array<int64_t, 3> workgroupSizes;
+ for (auto [index, attr] : llvm::enumerate(workgroupSizeAttr.value())) {
+ workgroupSizes[index] = llvm::cast<IntegerAttr>(attr).getInt();
+ }
+ return verifyLoweringConfiguration(moduleOp, translationInfo,
+ workgroupSizes, verifyGPUMatmulPipeline);
+ }
+ return success();
+}
+
+void LLVMGPUSelectLoweringStrategyPass::runOnOperation() {
+ IREE::HAL::ExecutableVariantOp variantOp = getOperation();
+ ModuleOp moduleOp = variantOp.getInnerModule();
+
+ if (failed(initGPULaunchConfig(moduleOp))) {
+ return signalPassFailure();
+ }
+
+ // There might be multiple entry points in the module. Currently, all of
+ // them need to have the same pipeline.
+ // TODO(ravishankarm): This is strange that this is not enforced
+ // structurally, but something to address later on. For now this restriction
+ // is fine.
+ llvm::StringMap<IREE::HAL::ExecutableExportOp> exportOps =
+ getAllEntryPoints(moduleOp);
+ std::optional<IREE::Codegen::TranslationInfoAttr> translationInfo;
+ for (auto &it : exportOps) {
+ auto exportOp = it.second;
+ if (IREE::Codegen::TranslationInfoAttr currTranslationInfo =
+ getTranslationInfo(exportOp)) {
+ if (translationInfo) {
+ if (currTranslationInfo != translationInfo.value()) {
+ moduleOp.emitOpError(
+ "unhandled compilation of entry point functions with different "
+ "translation info");
+ }
+ } else {
+ translationInfo = currTranslationInfo;
+ }
+
+ // Verify the properties of each entry point based on the target
+ // pipeline.
+ if (failed(verifyEntryPoint(moduleOp, currTranslationInfo, exportOp))) {
+ return signalPassFailure();
+ }
+ }
+ }
+}
+
+std::unique_ptr<OperationPass<IREE::HAL::ExecutableVariantOp>>
+createLLVMGPUSelectLoweringStrategyPass() {
+ return std::make_unique<LLVMGPUSelectLoweringStrategyPass>();
+}
+
+} // namespace iree_compiler
+} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.cpp
index 048c2fd..556b36d 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.cpp
@@ -593,8 +593,13 @@
passManager.addPass(createDropSchedulePass());
}
-void buildLLVMGPUTransformPassPipeline(OpPassManager &pm, bool useROCM) {
+void buildLLVMGPUCodegenStrategyInitializationPassPipeline(OpPassManager &pm) {
addCommonTargetExecutablePreprocessingPasses(pm);
+ pm.addPass(createLLVMGPUSelectLoweringStrategyPass());
+}
+
+void buildLLVMGPUCodegenPassPipeline(OpPassManager &pm, bool useROCM) {
+ buildLLVMGPUCodegenStrategyInitializationPassPipeline(pm);
pm.addPass(createLLVMGPULowerExecutableTargetPass());
OpPassManager &nestedModulePM = pm.nest<ModuleOp>();
//===--------------------------------------------------------------------===//
@@ -630,14 +635,14 @@
"iree-codegen-linalg-to-nvvm-pipeline",
"Runs the progressive lowering pipeline from Linalg to NVVM",
[](OpPassManager &passManager) {
- buildLLVMGPUTransformPassPipeline(passManager, false);
+ buildLLVMGPUCodegenPassPipeline(passManager, false);
});
static PassPipelineRegistration<> LinalgROCDLPipeline(
"iree-codegen-linalg-to-rocdl-pipeline",
"Runs the progressive lowering pipeline from Linalg to ROCDL",
[](OpPassManager &passManager) {
- buildLLVMGPUTransformPassPipeline(passManager, true);
+ buildLLVMGPUCodegenPassPipeline(passManager, true);
});
}
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.h b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.h
index c2c5245..c903f09 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.h
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.h
@@ -56,10 +56,13 @@
/// Default pass pipeline on GPU, currently used only for the ukernel path.
void addGPUDefaultPassPipeline(OpPassManager &pm);
+/// Populates passes needed to preprocess and select the strategy for lowering.
+void buildLLVMGPUCodegenStrategyInitializationPassPipeline(OpPassManager &pm);
+
/// Populates passes needed to lower a XLA HLO op to NVVM/ROCDL dialect via
/// the structured ops path. The pass manager `pm` in here should operate on
/// the module within the IREE::HAL::ExecutableOp.
-void buildLLVMGPUTransformPassPipeline(OpPassManager &pm, bool useROCM);
+void buildLLVMGPUCodegenPassPipeline(OpPassManager &pm, bool useROCM);
/// Performs the final conversion to NNVM+LLVM dialect.
std::unique_ptr<OperationPass<ModuleOp>> createConvertToNVVMPass();
@@ -73,6 +76,10 @@
std::unique_ptr<OperationPass<func::FuncOp>> createLLVMGPUDistribute();
+/// Create pass selecting the lowering strategy for LLVMGPU.
+std::unique_ptr<OperationPass<IREE::HAL::ExecutableVariantOp>>
+createLLVMGPUSelectLoweringStrategyPass();
+
/// Create pass calling the dynamic pipeline for LLVMGPU.
std::unique_ptr<OperationPass<IREE::HAL::ExecutableVariantOp>>
createLLVMGPULowerExecutableTargetPass();
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.td b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.td
index 01f72d6..04f2e6f 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.td
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.td
@@ -47,7 +47,6 @@
let constructor = "mlir::iree_compiler::createLLVMGPUCastAddressSpaceFunction()";
}
-// TODO: Bring the argument in line with the names used elsewhere.
def LLVMGPULowerExecutableTarget :
Pass<"iree-llvmgpu-lower-executable-target", "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
let summary = "Perform lowering of executable target using one of the IREE::HAL::DispatchLoweringPassPipeline";
@@ -60,6 +59,12 @@
let constructor = "mlir::iree_compiler::createLLVMGPUPackSharedMemoryAlloc()";
}
+def LLVMGPUSelectLoweringStrategy :
+ Pass<"iree-llvmgpu-select-lowering-strategy", "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
+ let summary = "Select a IREE::HAL::DispatchLoweringPassPipeline for lowering the target variant";
+ let constructor = "mlir::iree_compiler::createLLVMGPUSelectLoweringStrategyPass()";
+}
+
def LLVMGPUTensorCoreVectorization :
Pass<"iree-llvmgpu-tensorcore-vectorization", "func::FuncOp"> {
let summary = "Pass to convert linalg into Vector and transform it to a form that can be lowered to GPU MMA ops";
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/attention.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/attention.mlir
index 19ceac5..313a3f1 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/attention.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/attention.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt %s --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmgpu-lower-executable-target)))' \
+// RUN: iree-opt %s --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))' \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-jit=false \
// RUN: --iree-codegen-use-transform-dialect-strategy=%s | \
// RUN: FileCheck --check-prefix=CHECK %s
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/conv_pipeline_test.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/conv_pipeline_test.mlir
index 82995d9..7050437 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/conv_pipeline_test.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/conv_pipeline_test.mlir
@@ -1,5 +1,5 @@
// RUN: iree-opt --split-input-file \
-// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target,canonicalize)))' \
+// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target,canonicalize)))' \
// RUN: %s | FileCheck %s
#device_target_cuda = #hal.device.target<"cuda", {executable_targets = [#hal.executable.target<"cuda", "cuda-nvptx-fb", {target_arch = "sm_60"}>]}>
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/elementwise_pipeline.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/elementwise_pipeline.mlir
index e77dd68..9e02ec1 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/elementwise_pipeline.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/elementwise_pipeline.mlir
@@ -1,5 +1,4 @@
-
-// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
hal.executable @warp_reduction_dispatch {
hal.executable.variant public @cuda_nvptx_fb target(<"cuda", "cuda-nvptx-fb", {target_arch = "sm_60"}>) {
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/gpu_set_num_workgroups.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/gpu_set_num_workgroups.mlir
index c6b7399..6a109d5 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/gpu_set_num_workgroups.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/gpu_set_num_workgroups.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" \
+// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmgpu-select-lowering-strategy)))" \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-jit=false %s | FileCheck %s
// Transform dialect attributes are tested separately.
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/illegal_configuration.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/illegal_configuration.mlir
index 6db7807..fa59d7c 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/illegal_configuration.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/illegal_configuration.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration=true})))" --verify-diagnostics --split-input-file %s
+// RUN: iree-opt --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" --verify-diagnostics --split-input-file %s
#config = #iree_codegen.lowering_config<tile_sizes = []>
#translation = #iree_codegen.translation_info<LLVMGPUMatmulSimt>
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/linalg_transform.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/linalg_transform.mlir
index 738c009..e6c4d1b 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/linalg_transform.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/linalg_transform.mlir
@@ -1,9 +1,9 @@
-// RUN: iree-opt %s --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmgpu-lower-executable-target)))" \
+// RUN: iree-opt %s --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))" \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-jit=false \
// RUN: --iree-codegen-use-transform-dialect-strategy=%p/transform_dialect_codegen_bufferize_spec.mlir | \
// RUN: FileCheck %s
-// RUN: iree-opt %s --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmgpu-lower-executable-target)))" \
+// RUN: iree-opt %s --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))" \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-jit=false \
// RUN: --iree-codegen-use-transform-dialect-strategy=%p/transform_dialect_codegen_foreach_to_gpu_spec.mlir | \
// RUN: FileCheck %s --check-prefix=FOREACH-TO-GPU
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/pack_pipeline_test.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/pack_pipeline_test.mlir
index b7f2235..181e4ad 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/pack_pipeline_test.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/pack_pipeline_test.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_cuda.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_cuda.mlir
index 20d4ac7..467a964 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_cuda.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_cuda.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(builtin.module(func.func(iree-linalg-ext-decompose-softmax)), iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(builtin.module(func.func(iree-linalg-ext-decompose-softmax)), iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_rocm.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_rocm.mlir
index 20213aa..9aadf2b 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_rocm.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_rocm.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(builtin.module(func.func(iree-linalg-ext-decompose-softmax)), iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(builtin.module(func.func(iree-linalg-ext-decompose-softmax)), iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_transform_cuda.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_transform_cuda.mlir
index 774522a..72be1f0 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_transform_cuda.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_transform_cuda.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
hal.executable @small_reduction {
hal.executable.variant public @cuda_nvptx_fb target(<"cuda", "cuda-nvptx-fb", {target_arch = "sm_60"}>) {
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_transform_rocm.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_transform_rocm.mlir
index c75c3db..765312b 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_transform_rocm.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/reduction_pipeline_transform_rocm.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
hal.executable @group_reduction_1d {
hal.executable.variant public @rocm_hsaco_fb target(<"rocm", "rocm-hsaco-fb", {target_arch = "gfx1100"}>) {
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_batch_matmul.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_batch_matmul.mlir
index d127328..edbbfa4 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_batch_matmul.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_batch_matmul.mlir
@@ -1,8 +1,8 @@
-// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" \
+// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-jit=1 --iree-codegen-llvmgpu-enable-transform-dialect-batch-matmul-strategy |\
// RUN: FileCheck %s --check-prefixes=CHECK,DEFAULT
-// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" \
+// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-jit=1 --iree-codegen-llvmgpu-enable-transform-dialect-batch-matmul-strategy \
// RUN: -td-matmul-strategy-blk-sizes=128,64,32,2 \
// RUN: -td-matmul-strategy-reduc-size=8 \
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_convolution.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_convolution.mlir
index b634f69..0c18806 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_convolution.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_convolution.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" --iree-codegen-llvmgpu-enable-transform-dialect-implicit-gemm-strategy | FileCheck %s
+// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" --iree-codegen-llvmgpu-enable-transform-dialect-implicit-gemm-strategy | FileCheck %s
hal.executable @nchw_convolution {
hal.executable.variant public @cuda_nvptx_fb target(<"cuda", "cuda-nvptx-fb", {target_arch = "sm_80"}>) {
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_matmul.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_matmul.mlir
index 267886e..1c8017d 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_matmul.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_matmul.mlir
@@ -1,8 +1,8 @@
-// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" --iree-codegen-llvmgpu-enable-transform-dialect-aligned-matmul | FileCheck %s
+// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" --iree-codegen-llvmgpu-enable-transform-dialect-aligned-matmul | FileCheck %s
// Check that setting the command line options affect the transform
// strategy as expected.
-// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" \
+// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" \
// RUN: -td-matmul-strategy-blk-sizes=256,64,1 \
// RUN: -td-matmul-strategy-reduc-size=8 \
// RUN: -td-matmul-strategy-num-threads=32,4,1 \
@@ -13,7 +13,7 @@
// RUN: | FileCheck --check-prefix=WITH_OPTIONS %s
// Check that various more exotic strategies apply properly e2e but without otherwise checking their content.
-// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" \
+// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-aligned-matmul \
// RUN: -td-matmul-strategy-blk-sizes=16,16,1 \
// RUN: -td-matmul-strategy-reduc-size=16 \
@@ -25,7 +25,7 @@
// RUN: | FileCheck --check-prefix=WITH_OPTIONS_2 %s
// Check that various more exotic strategies apply properly e2e but without otherwise checking their content.
-// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" \
+// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-aligned-matmul \
// RUN: -td-matmul-strategy-blk-sizes=128,64,1 \
// RUN: -td-matmul-strategy-reduc-size=16 \
@@ -36,7 +36,7 @@
// RUN: -td-matmul-strategy-pipeline-depth=3 \
// RUN: | FileCheck --check-prefix=WITH_OPTIONS_3 %s
-// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" --iree-codegen-llvmgpu-enable-transform-dialect-small-matmul \
+// RUN: iree-opt %s --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" --iree-codegen-llvmgpu-enable-transform-dialect-small-matmul \
// RUN: | FileCheck --check-prefix=SMALL %s
hal.executable @matmul_1 {
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_pad.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_pad.mlir
index 06253af..5168e75 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_pad.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/set_transform_strategy_pad.mlir
@@ -1,12 +1,12 @@
// RUN: iree-opt %s --split-input-file \
-// RUN: --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" \
+// RUN: --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-pad-strategy \
// RUN: | FileCheck %s
// Check that setting the command line options affect the transform
// strategy as expected.
// RUN: iree-opt %s --split-input-file \
-// RUN: --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target{test-lowering-configuration})))" \
+// RUN: --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy)))" \
// RUN: --iree-codegen-llvmgpu-enable-transform-dialect-pad-strategy \
// RUN: --td-pad-strategy-blk-sizes=16,32,1 \
// RUN: --td-pad-strategy-num-threads=8,4,1 \
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/transpose_pipeline_test.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/transpose_pipeline_test.mlir
index 89126da..b4d2087 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/transpose_pipeline_test.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/transpose_pipeline_test.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target, fold-memref-alias-ops, canonicalize, cse)))" %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target, fold-memref-alias-ops, canonicalize, cse)))" %s | FileCheck %s
#device_target_cuda = #hal.device.target<"cuda", {executable_targets = [#hal.executable.target<"cuda", "cuda-nvptx-fb", {target_arch = "sm_80"}>]}>
#executable_target_cuda_nvptx_fb = #hal.executable.target<"cuda", "cuda-nvptx-fb", {target_arch = "sm_80"}>
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/workgroup_specialization_pipeline_test.mlir b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/workgroup_specialization_pipeline_test.mlir
index 981d874..8a4e5a1 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/test/workgroup_specialization_pipeline_test.mlir
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/test/workgroup_specialization_pipeline_test.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmgpu-select-lowering-strategy, iree-llvmgpu-lower-executable-target)))" %s | FileCheck %s
module attributes {hal.device.targets = [#hal.device.target<"cuda", {executable_targets = [#hal.executable.target<"cuda", "cuda-nvptx-fb", {target_arch = "sm_80"}>]}>]} {
hal.executable private @forward_dispatch_116 {
hal.executable.variant public @cuda_nvptx_fb target(<"cuda", "cuda-nvptx-fb", {target_arch = "sm_80"}>) {
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel b/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel
index 23ac1dc..1069395 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel
@@ -66,6 +66,7 @@
"SPIRVInitialVectorLowering.cpp",
"SPIRVLowerExecutableTargetPass.cpp",
"SPIRVMapMemRefStorageClass.cpp",
+ "SPIRVSelectLoweringStrategy.cpp",
"SPIRVTile.cpp",
"SPIRVTileAndDistribute.cpp",
"SPIRVTileAndPromote.cpp",
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt
index e722a23..c23a25a 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt
@@ -65,6 +65,7 @@
"SPIRVInitialVectorLowering.cpp"
"SPIRVLowerExecutableTargetPass.cpp"
"SPIRVMapMemRefStorageClass.cpp"
+ "SPIRVSelectLoweringStrategy.cpp"
"SPIRVTile.cpp"
"SPIRVTileAndDistribute.cpp"
"SPIRVTileAndPromote.cpp"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp
index 86e82fc..348f625 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp
@@ -654,11 +654,16 @@
// Entry Point
//===----------------------------------------------------------------------===//
-void buildSPIRVCodegenPassPipeline(OpPassManager &pm, bool enableFastMath) {
+void buildSPIRVCodegenStrategyInitializationPassPipeline(OpPassManager &pm) {
addCommonTargetExecutablePreprocessingPasses(pm);
auto &nestedModulePM = pm.nest<ModuleOp>();
nestedModulePM.addNestedPass<func::FuncOp>(
createSPIRVGeneralizeNamedOpsPass());
+ pm.addPass(createSPIRVSelectLoweringStrategyPass());
+}
+
+void buildSPIRVCodegenPassPipeline(OpPassManager &pm, bool enableFastMath) {
+ buildSPIRVCodegenStrategyInitializationPassPipeline(pm);
pm.addPass(createSPIRVLowerExecutableTargetPass());
addMemRefLoweringPasses(pm.nest<ModuleOp>());
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h
index e762e11..f00fd01 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h
@@ -50,6 +50,10 @@
///
void addSPIRVWinogradVectorizePassPipeline(OpPassManager &pm);
+/// Populates passes needed to preprocess the input variant before lowering
+/// and select lowering strategies.
+void buildSPIRVCodegenStrategyInitializationPassPipeline(OpPassManager &pm);
+
/// Populates passes needed to lower linalg/arith/math ops to SPIR-V ops via
/// the structured ops path. The pass manager `pm` here operate on the module
/// within the IREE::HAL::ExecutableOp.
@@ -99,6 +103,10 @@
std::unique_ptr<OperationPass<func::FuncOp>>
createSPIRVGeneralizeNamedOpsPass();
+/// Pass to set the lowering strategy for the target variant.
+std::unique_ptr<OperationPass<IREE::HAL::ExecutableVariantOp>>
+createSPIRVSelectLoweringStrategyPass();
+
/// Main pass to lower executables to scalar + vector code on SPIR-V path.
/// Invokes one of the pass pipelines that translate the executable to
/// scalar + vector code.
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td
index e409e8b..aef37e9 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td
@@ -79,6 +79,15 @@
let constructor = "mlir::iree_compiler::createSPIRVMapMemRefStorageClassPass()";
}
+def SPIRVSelectLoweringStrategy :
+ Pass<"iree-spirv-select-lowering-strategy-pass",
+ "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
+ let summary = "Select the IREE::HAL::DispatchLoweringPassPipeline for lowering"
+ "to SPIR-V";
+ let constructor =
+ "mlir::iree_compiler::createSPIRVSelectLoweringStrategyPass()";
+}
+
def SPIRVTile : Pass<"iree-spirv-tile", "func::FuncOp"> {
let summary = "Tile Linalg ops with tensor semantics to invocations";
let constructor = "mlir::iree_compiler::createSPIRVTilePass()";
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp
index 1613b2d..9417930 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp
@@ -54,89 +54,19 @@
}
void runOnOperation() override;
-
-private:
- Option<bool> testLoweringConfiguration{
- *this, "test-lowering-configuration",
- llvm::cl::desc("Flag used for lit-testing the configuration set for root "
- "ops in hal.executable.variants. Defaults to false. Set "
- "to true for lit tests; not for general usage"),
- llvm::cl::init(false)};
};
} // namespace
-/// Verify that valid configuration is set for all ops within the compiled
-/// module.
-template <typename F>
-static LogicalResult
-verifyLoweringConfiguration(ModuleOp module,
- IREE::Codegen::TranslationInfoAttr translationInfo,
- ArrayRef<int64_t> workgroupSize, F verificationFn) {
- auto walkResult = module.walk([&](Operation *op) -> WalkResult {
- IREE::Codegen::LoweringConfigAttr loweringConfig = getLoweringConfig(op);
- if (!loweringConfig)
- return WalkResult::advance();
- return verificationFn(op, loweringConfig, translationInfo, workgroupSize);
- });
- return failure(walkResult.wasInterrupted());
-}
-
-static LogicalResult
-verifyEntryPoint(ModuleOp moduleOp,
- IREE::Codegen::TranslationInfoAttr translationInfo,
- IREE::HAL::ExecutableExportOp exportOp) {
- if (translationInfo.getDispatchLoweringPassPipeline() ==
- CodeGenPipeline::TransformDialectCodegen) {
- // Transform dialect encodes configuration into the schedule directly.
- return success();
- }
-
- std::optional<mlir::ArrayAttr> workgroupSizeAttr =
- exportOp.getWorkgroupSize();
- if (!workgroupSizeAttr || workgroupSizeAttr->size() != 3) {
- return moduleOp.emitError(
- "expected workgroup size to have three dimensions for SPIR-V "
- "pipelines");
- }
-
- std::array<int64_t, 3> workgroupSizes;
- for (auto [index, attr] : llvm::enumerate(workgroupSizeAttr.value())) {
- workgroupSizes[index] = llvm::cast<IntegerAttr>(attr).getInt();
- }
-
- switch (translationInfo.getDispatchLoweringPassPipeline()) {
- case CodeGenPipeline::SPIRVBaseVectorize:
- return verifyLoweringConfiguration(moduleOp, translationInfo,
- workgroupSizes,
- verifySPIRVBaseVectorizePassPipeline);
- case CodeGenPipeline::SPIRVMatmulPromoteVectorize:
- return verifyLoweringConfiguration(
- moduleOp, translationInfo, workgroupSizes,
- verifySPIRVMatmulPromoteVectorizePassPipeline);
- case CodeGenPipeline::SPIRVCooperativeMatrixVectorize:
- return verifyLoweringConfiguration(
- moduleOp, translationInfo, workgroupSizes,
- verifySPIRVCooperativeMatrixVectorizePassPipeline);
- default:
- break;
- }
- return success();
-}
-
void SPIRVLowerExecutableTargetPass::runOnOperation() {
IREE::HAL::ExecutableVariantOp variantOp = getOperation();
ModuleOp moduleOp = variantOp.getInnerModule();
OpPassManager pipeline(IREE::HAL::ExecutableVariantOp::getOperationName());
- if (failed(initSPIRVLaunchConfig(moduleOp))) {
- return signalPassFailure();
- }
// There might be multiple entry points in the module. Currently, all of
- // them need to have the same pipeline.
- // TODO(ravishankarm): This is strange that this is not enforced
- // structurally, but something to address later on. For now this restriction
- // is fine.
+ // them need to have the same pipeline. This should have been verified
+ // when setting the strategy, however we still need to retrieve the
+ // translation info due to this restriction.
llvm::StringMap<IREE::HAL::ExecutableExportOp> exportOps =
getAllEntryPoints(moduleOp);
std::optional<IREE::Codegen::TranslationInfoAttr> translationInfo;
@@ -153,18 +83,11 @@
}
continue;
}
-
- // Verify the properties of each entry point based on the target
- // pipeline.
- if (failed(verifyEntryPoint(moduleOp, currTranslationInfo, exportOp))) {
- return signalPassFailure();
- }
-
translationInfo = currTranslationInfo;
}
}
- if (!testLoweringConfiguration && translationInfo.has_value()) {
+ if (translationInfo.has_value()) {
switch (translationInfo.value().getDispatchLoweringPassPipeline()) {
case CodeGenPipeline::SPIRVBaseLowering:
addSPIRVBaseLoweringPassPipeline(pipeline);
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVSelectLoweringStrategy.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVSelectLoweringStrategy.cpp
new file mode 100644
index 0000000..2477c08
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVSelectLoweringStrategy.cpp
@@ -0,0 +1,169 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
+#include "iree/compiler/Codegen/Dialect/IREECodegenAttrs.h"
+#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
+#include "iree/compiler/Codegen/SPIRV/KernelConfig.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
+#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
+#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
+#include "llvm/Support/Debug.h"
+#include "mlir/Dialect/Affine/IR/AffineOps.h"
+#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
+#include "mlir/Dialect/GPU/IR/GPUDialect.h"
+#include "mlir/Dialect/MemRef/Transforms/Transforms.h"
+#include "mlir/Dialect/SPIRV/IR/SPIRVDialect.h"
+#include "mlir/Dialect/Transform/IR/TransformDialect.h"
+#include "mlir/Dialect/Vector/IR/VectorOps.h"
+#include "mlir/Pass/Pass.h"
+#include "mlir/Pass/PassManager.h"
+#include "mlir/Pass/PassRegistry.h"
+#include "mlir/Transforms/Passes.h"
+
+#define DEBUG_TYPE "iree-spirv-select-lowering-strategy-pass"
+
+namespace mlir {
+namespace iree_compiler {
+
+using CodeGenPipeline = IREE::Codegen::DispatchLoweringPassPipeline;
+
+namespace {
+/// Lowers a hal.executable.variant inner module to SPIR-V scalar/native-vector
+/// code. Invokes different compilation pipeline to
+/// - first lower to scalar/native-vector code,
+/// - then convert to SPIRV dialect.
+class SPIRVSelectLoweringStrategyPass
+ : public SPIRVSelectLoweringStrategyBase<SPIRVSelectLoweringStrategyPass> {
+public:
+ SPIRVSelectLoweringStrategyPass() = default;
+ SPIRVSelectLoweringStrategyPass(const SPIRVSelectLoweringStrategyPass &pass) {
+ }
+
+ void getDependentDialects(DialectRegistry ®istry) const override {
+ // TODO(qedawkins): Once TransformStrategies is deprecated, drop the
+ // unnecessary dialect registrations.
+ registry
+ .insert<IREE::Codegen::IREECodegenDialect, affine::AffineDialect,
+ gpu::GPUDialect, IREE::HAL::HALDialect, linalg::LinalgDialect,
+ IREE::LinalgExt::IREELinalgExtDialect, memref::MemRefDialect,
+ bufferization::BufferizationDialect, scf::SCFDialect,
+ spirv::SPIRVDialect, transform::TransformDialect,
+ vector::VectorDialect>();
+ }
+
+ void runOnOperation() override;
+};
+} // namespace
+
+/// Verify that valid configuration is set for all ops within the compiled
+/// module.
+template <typename F>
+static LogicalResult
+verifyLoweringConfiguration(ModuleOp module,
+ IREE::Codegen::TranslationInfoAttr translationInfo,
+ ArrayRef<int64_t> workgroupSize, F verificationFn) {
+ auto walkResult = module.walk([&](Operation *op) -> WalkResult {
+ IREE::Codegen::LoweringConfigAttr loweringConfig = getLoweringConfig(op);
+ if (!loweringConfig)
+ return WalkResult::advance();
+ return verificationFn(op, loweringConfig, translationInfo, workgroupSize);
+ });
+ return failure(walkResult.wasInterrupted());
+}
+
+static LogicalResult
+verifyEntryPoint(ModuleOp moduleOp,
+ IREE::Codegen::TranslationInfoAttr translationInfo,
+ IREE::HAL::ExecutableExportOp exportOp) {
+ if (translationInfo.getDispatchLoweringPassPipeline() ==
+ CodeGenPipeline::TransformDialectCodegen) {
+ // Transform dialect encodes configuration into the schedule directly.
+ return success();
+ }
+
+ std::optional<mlir::ArrayAttr> workgroupSizeAttr =
+ exportOp.getWorkgroupSize();
+ if (!workgroupSizeAttr || workgroupSizeAttr->size() != 3) {
+ return moduleOp.emitError(
+ "expected workgroup size to have three dimensions for SPIR-V "
+ "pipelines");
+ }
+
+ std::array<int64_t, 3> workgroupSizes;
+ for (auto [index, attr] : llvm::enumerate(workgroupSizeAttr.value())) {
+ workgroupSizes[index] = llvm::cast<IntegerAttr>(attr).getInt();
+ }
+
+ switch (translationInfo.getDispatchLoweringPassPipeline()) {
+ case CodeGenPipeline::SPIRVBaseVectorize:
+ return verifyLoweringConfiguration(moduleOp, translationInfo,
+ workgroupSizes,
+ verifySPIRVBaseVectorizePassPipeline);
+ case CodeGenPipeline::SPIRVMatmulPromoteVectorize:
+ return verifyLoweringConfiguration(
+ moduleOp, translationInfo, workgroupSizes,
+ verifySPIRVMatmulPromoteVectorizePassPipeline);
+ case CodeGenPipeline::SPIRVCooperativeMatrixVectorize:
+ return verifyLoweringConfiguration(
+ moduleOp, translationInfo, workgroupSizes,
+ verifySPIRVCooperativeMatrixVectorizePassPipeline);
+ default:
+ break;
+ }
+ return success();
+}
+
+void SPIRVSelectLoweringStrategyPass::runOnOperation() {
+ IREE::HAL::ExecutableVariantOp variantOp = getOperation();
+ ModuleOp moduleOp = variantOp.getInnerModule();
+
+ OpPassManager pipeline(IREE::HAL::ExecutableVariantOp::getOperationName());
+
+ if (failed(initSPIRVLaunchConfig(moduleOp))) {
+ return signalPassFailure();
+ }
+ // There might be multiple entry points in the module. Currently, all of
+ // them need to have the same pipeline.
+ // TODO(ravishankarm): This is strange that this is not enforced
+ // structurally, but something to address later on. For now this restriction
+ // is fine.
+ llvm::StringMap<IREE::HAL::ExecutableExportOp> exportOps =
+ getAllEntryPoints(moduleOp);
+ std::optional<IREE::Codegen::TranslationInfoAttr> translationInfo;
+ for (auto &it : exportOps) {
+ auto exportOp = it.second;
+ if (IREE::Codegen::TranslationInfoAttr currTranslationInfo =
+ getTranslationInfo(exportOp)) {
+ if (translationInfo) {
+ if (currTranslationInfo != translationInfo.value()) {
+ moduleOp.emitError(
+ "unhandled compilation of entry point function with different "
+ "translation info within a module");
+ return signalPassFailure();
+ }
+ continue;
+ }
+
+ // Verify the properties of each entry point based on the target
+ // pipeline.
+ if (failed(verifyEntryPoint(moduleOp, currTranslationInfo, exportOp))) {
+ return signalPassFailure();
+ }
+
+ translationInfo = currTranslationInfo;
+ }
+ }
+}
+
+std::unique_ptr<OperationPass<IREE::HAL::ExecutableVariantOp>>
+createSPIRVSelectLoweringStrategyPass() {
+ return std::make_unique<SPIRVSelectLoweringStrategyPass>();
+}
+
+} // namespace iree_compiler
+} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_adreno_conv.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_adreno_conv.mlir
index 5ea416a..ee8b5b3 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_adreno_conv.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_adreno_conv.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
// Conv - large OC - distribute to only one workgroup dimension.
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_adreno_matmul.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_adreno_matmul.mlir
index f060fbb..ee57bda 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_adreno_matmul.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_adreno_matmul.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
// Large matmul that can match the best tiling scheme.
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_conv.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_conv.mlir
index 8541ec8..66fed52 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_conv.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_conv.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_matmul.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_matmul.mlir
index 84315bb..a0290a2 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_matmul.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_matmul.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_matmul_cooperative_ops.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_matmul_cooperative_ops.mlir
index 9a3d937..7920007 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_matmul_cooperative_ops.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_amd_matmul_cooperative_ops.mlir
@@ -1,5 +1,5 @@
// RUN: iree-opt --split-input-file \
-// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' \
+// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' \
// RUN: %s | FileCheck %s
#map = affine_map<(d0, d1) -> (d0, d1)>
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_conv.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_conv.mlir
index 3af4885..2bcc068 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_conv.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_conv.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
// Convolution with consumer pointwise ops
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_linalg_ext_ops.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_linalg_ext_ops.mlir
index 321eb6d..271c36b 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_linalg_ext_ops.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_linalg_ext_ops.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_linalg_ops.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_linalg_ops.mlir
index 38c6574..96e0457 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_linalg_ops.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_linalg_ops.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 2, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_matmul.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_matmul.mlir
index a7566bd..0d7c3ab 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_matmul.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_matmul.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
// Odd K that forbids vectorization.
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_matvec.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_matvec.mlir
index 16c9346..5a92011 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_matvec.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_matvec.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_reduction.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_reduction.mlir
index 891cf7b..a4630d7 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_reduction.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_reduction.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_sub_byte_types.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_sub_byte_types.mlir
index 60852a3..ad0433c 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_sub_byte_types.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_default_sub_byte_types.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_mali_conv.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_mali_conv.mlir
index 32edc09..bb24848 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_mali_conv.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_mali_conv.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
// Conv - large OC - distribute to only one workgroup dimension.
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_mali_matmul.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_mali_matmul.mlir
index 86492a2..6651e26 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_mali_matmul.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_mali_matmul.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
// Large matmul that can match the best tiling scheme.
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_nvidia_matmul.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_nvidia_matmul.mlir
index 781c618..04f8929 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_nvidia_matmul.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_nvidia_matmul.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_nvidia_matmul_cooperative_ops.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_nvidia_matmul_cooperative_ops.mlir
index 34c6525..1bebbc8 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_nvidia_matmul_cooperative_ops.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_nvidia_matmul_cooperative_ops.mlir
@@ -1,5 +1,5 @@
// RUN: iree-opt --split-input-file \
-// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' \
+// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))' \
// RUN: %s | FileCheck %s
#map = affine_map<(d0, d1) -> (d0, d1)>
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_user.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_user.mlir
index 045bc3e..1c6d4ca 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/config_user.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/config_user.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-spirv-select-lowering-strategy-pass)))' %s | FileCheck %s
#compilation = #iree_codegen.compilation_info<
lowering_config = <tile_sizes = [[128, 256], [16, 16]]>,
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/illegal_configuration.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/illegal_configuration.mlir
index 770adde..b2c56e3 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/illegal_configuration.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/illegal_configuration.mlir
@@ -1,5 +1,5 @@
// RUN: iree-opt \
-// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-spirv-lower-executable-target-pass{test-lowering-configuration=true})))' \
+// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-spirv-select-lowering-strategy-pass)))' \
// RUN: --verify-diagnostics --split-input-file %s
#compilation = #iree_codegen.compilation_info<
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matmul_fusion.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matmul_fusion.mlir
index 1eea6f6..b3ca53f 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matmul_fusion.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matmul_fusion.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-spirv-lower-executable-target-pass)))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-spirv-select-lowering-strategy-pass, iree-spirv-lower-executable-target-pass)))' %s | FileCheck %s
#compilation = #iree_codegen.compilation_info<
lowering_config = <tile_sizes = [[32, 128, 1, 32]]>,
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matmul_promotion.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matmul_promotion.mlir
index 2462ce3..a571389 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matmul_promotion.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matmul_promotion.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-spirv-lower-executable-target-pass)))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-spirv-select-lowering-strategy-pass, iree-spirv-lower-executable-target-pass)))' %s | FileCheck %s
// Verify pipelining + multi-buffering.
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matvec.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matvec.mlir
index 3f7e9d5..bab1453 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matvec.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_matvec.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass)))' %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass, iree-spirv-lower-executable-target-pass)))' %s | FileCheck %s
#pipeline_layout = #hal.pipeline.layout<push_constants = 0, sets = [
#hal.descriptor_set.layout<0, bindings = [
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_reduction.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_reduction.mlir
index 6329dd8..7d4a2d5 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_reduction.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_reduction.mlir
@@ -1,5 +1,5 @@
// RUN: iree-opt --split-input-file \
-// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(builtin.module(func.func(iree-linalg-ext-decompose-softmax)), iree-spirv-lower-executable-target-pass)))' \
+// RUN: --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(builtin.module(func.func(iree-linalg-ext-decompose-softmax)), iree-spirv-select-lowering-strategy-pass, iree-spirv-lower-executable-target-pass)))' \
// RUN: %s | FileCheck %s
#executable_target_vulkan_spirv_fb = #hal.executable.target<"vulkan", "vulkan-spirv-fb", {
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_scalar_dispatch.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_scalar_dispatch.mlir
index 34ca23d..5cb0fc5 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_scalar_dispatch.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/lowering_scalar_dispatch.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass)))' -mlir-print-local-scope %s | FileCheck %s
+// RUN: iree-opt --split-input-file --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass, iree-spirv-lower-executable-target-pass)))' -mlir-print-local-scope %s | FileCheck %s
#executable_target_vulkan_spirv_fb = #hal.executable.target<"vulkan", "vulkan-spirv-fb", {
spirv.target_env = #spirv.target_env<#spirv.vce<v1.5, [Shader], []>, Unknown:Unknown,
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/test/set_transform_strategy.mlir b/compiler/src/iree/compiler/Codegen/SPIRV/test/set_transform_strategy.mlir
index 58b7ac5..300398f 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/test/set_transform_strategy.mlir
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/test/set_transform_strategy.mlir
@@ -1,5 +1,5 @@
// RUN: iree-opt %s --split-input-file \
-// RUN: --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-spirv-lower-executable-target-pass{test-lowering-configuration})))"\
+// RUN: --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-spirv-select-lowering-strategy-pass)))"\
// RUN: --iree-spirv-enable-transform-dialect-jit=true | FileCheck %s
hal.executable @matmul {
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/test/pipeline.mlir b/compiler/src/iree/compiler/Codegen/VMVX/test/pipeline.mlir
index 52f28bf..5b1e09f 100644
--- a/compiler/src/iree/compiler/Codegen/VMVX/test/pipeline.mlir
+++ b/compiler/src/iree/compiler/Codegen/VMVX/test/pipeline.mlir
@@ -1,4 +1,4 @@
-// RUN: iree-opt --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-lower-executable-target)))" --split-input-file %s | FileCheck %s
+// RUN: iree-opt --pass-pipeline="builtin.module(hal.executable(hal.executable.variant(iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))" --split-input-file %s | FileCheck %s
hal.executable private @mmt4d_ukernel {
hal.executable.variant public @vmvx_bytecode_fb target(<"vmvx", "vmvx-bytecode-fb", {ukernels = true}>) {
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Target/ROCM/ROCMTarget.cpp b/compiler/src/iree/compiler/Dialect/HAL/Target/ROCM/ROCMTarget.cpp
index b72f31d..04e5578 100644
--- a/compiler/src/iree/compiler/Dialect/HAL/Target/ROCM/ROCMTarget.cpp
+++ b/compiler/src/iree/compiler/Dialect/HAL/Target/ROCM/ROCMTarget.cpp
@@ -112,7 +112,7 @@
if (variantOp.isExternal())
return;
- buildLLVMGPUTransformPassPipeline(passManager, true);
+ buildLLVMGPUCodegenPassPipeline(passManager, true);
}
LogicalResult serializeExecutable(const SerializationOptions &options,
diff --git a/compiler/src/iree/compiler/Dialect/VMVX/Transforms/Passes.cpp b/compiler/src/iree/compiler/Dialect/VMVX/Transforms/Passes.cpp
index d8c4d71..fcabd35 100644
--- a/compiler/src/iree/compiler/Dialect/VMVX/Transforms/Passes.cpp
+++ b/compiler/src/iree/compiler/Dialect/VMVX/Transforms/Passes.cpp
@@ -40,6 +40,7 @@
// TODO: Remove the following pass the plumb support for #hal.descriptor_type
// memory space through the stack.
passManager.addPass(createEraseHALDescriptorTypeFromMemRefPass());
+ passManager.addPass(createLLVMCPUSelectLoweringStrategyPass());
passManager.addPass(createLLVMCPULowerExecutableTargetPass());
OpPassManager &nestedModulePM = passManager.nest<ModuleOp>();
diff --git a/tests/transform_dialect/cpu/eltwise_reduction_eltwise.mlir b/tests/transform_dialect/cpu/eltwise_reduction_eltwise.mlir
index d105660..8718fa8 100644
--- a/tests/transform_dialect/cpu/eltwise_reduction_eltwise.mlir
+++ b/tests/transform_dialect/cpu/eltwise_reduction_eltwise.mlir
@@ -49,7 +49,7 @@
// RUN: --iree-flow-transformation-pipeline \
// RUN: --iree-stream-transformation-pipeline \
// RUN: --iree-hal-configuration-pipeline | \
-// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmcpu-lower-executable-target)))' \
+// RUN: iree-opt --pass-pipeline='builtin.module(hal.executable(hal.executable.variant(iree-codegen-materialize-user-configs, iree-llvmcpu-select-lowering-strategy, iree-llvmcpu-lower-executable-target)))' \
// RUN: --iree-codegen-llvmcpu-enable-transform-dialect-jit | \
// RUN: FileCheck %s
