[NFC] Move LLVMCPUVectorization pass to common GenericVectorization pass (#14228)
There are nothing CPU specific in the pass. It is just a pass that
converts LinAlg ops and tensor ops to vector ops. Move it to Common/
then we can reuse it for other backends.
diff --git a/compiler/src/iree/compiler/Codegen/Common/BUILD.bazel b/compiler/src/iree/compiler/Codegen/Common/BUILD.bazel
index 69c0317..c18a35e 100644
--- a/compiler/src/iree/compiler/Codegen/Common/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/Common/BUILD.bazel
@@ -162,6 +162,7 @@
"FoldTensorExtractOpPass.cpp",
"ForOpCanonicalizationPass.cpp",
"FuseTensorPadWithConsumer.cpp",
+ "GenericVectorization.cpp",
"HoistStaticallyBoundAllocations.cpp",
"IREEComprehensiveBufferizePass.cpp",
"IREEExpandStridedMetadata.cpp",
diff --git a/compiler/src/iree/compiler/Codegen/Common/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Common/CMakeLists.txt
index 532e535..f21bd13 100644
--- a/compiler/src/iree/compiler/Codegen/Common/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/Common/CMakeLists.txt
@@ -137,6 +137,7 @@
"FoldTensorExtractOpPass.cpp"
"ForOpCanonicalizationPass.cpp"
"FuseTensorPadWithConsumer.cpp"
+ "GenericVectorization.cpp"
"HoistStaticallyBoundAllocations.cpp"
"IREEComprehensiveBufferizePass.cpp"
"IREEExpandStridedMetadata.cpp"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorization.cpp b/compiler/src/iree/compiler/Codegen/Common/GenericVectorization.cpp
similarity index 93%
rename from compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorization.cpp
rename to compiler/src/iree/compiler/Codegen/Common/GenericVectorization.cpp
index 6abee23..4e48b58 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorization.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GenericVectorization.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
-#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
@@ -21,7 +21,7 @@
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
-#define DEBUG_TYPE "iree-llvmcpu-vectorization"
+#define DEBUG_TYPE "iree-codegen-generic-vectorization"
#define VEC_DBGS() (llvm::dbgs() << '[' << DEBUG_TYPE << "] ")
namespace mlir {
@@ -211,11 +211,11 @@
return vecSize;
}
-class LLVMCPUVectorizationPass
- : public LLVMCPUVectorizationBase<LLVMCPUVectorizationPass> {
+class GenericVectorizationPass
+ : public GenericVectorizationBase<GenericVectorizationPass> {
public:
- using LLVMCPUVectorizationBase::LLVMCPUVectorizationBase;
- LLVMCPUVectorizationPass(const LLVMCPUVectorizationPassOptions &options) {
+ using GenericVectorizationBase::GenericVectorizationBase;
+ GenericVectorizationPass(const GenericVectorizationPassOptions &options) {
this->enableVectorMasking.setValue(options.enableVectorMasking);
this->vectorizePadding.setValue(options.vectorizePadding);
this->vectorizeGatherAccesses.setValue(options.vectorizeGatherAccesses);
@@ -228,7 +228,7 @@
void runOnOperation() override;
};
-void LLVMCPUVectorizationPass::runOnOperation() {
+void GenericVectorizationPass::runOnOperation() {
MLIRContext *context = &getContext();
auto funcOp = getOperation();
SmallVector<int64_t> canonicalVectorShape;
@@ -303,12 +303,12 @@
}
} // namespace
-std::unique_ptr<OperationPass<func::FuncOp>> createLLVMCPUVectorizationPass() {
- return std::make_unique<LLVMCPUVectorizationPass>();
+std::unique_ptr<OperationPass<func::FuncOp>> createGenericVectorizationPass() {
+ return std::make_unique<GenericVectorizationPass>();
}
std::unique_ptr<OperationPass<func::FuncOp>>
-createLLVMCPUVectorizationPass(const LLVMCPUVectorizationPassOptions &options) {
- return std::make_unique<LLVMCPUVectorizationPass>(options);
+createGenericVectorizationPass(const GenericVectorizationPassOptions &options) {
+ return std::make_unique<GenericVectorizationPass>(options);
}
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/Common/Passes.h b/compiler/src/iree/compiler/Codegen/Common/Passes.h
index 165b378..3977041 100644
--- a/compiler/src/iree/compiler/Codegen/Common/Passes.h
+++ b/compiler/src/iree/compiler/Codegen/Common/Passes.h
@@ -125,6 +125,16 @@
std::unique_ptr<OperationPass<func::FuncOp>>
createFuseTensorPadWithConsumerPass();
+struct GenericVectorizationPassOptions {
+ bool enableVectorMasking = false;
+ bool vectorizePadding = false;
+ bool vectorizeGatherAccesses = false;
+};
+/// Creates a pass to perform vectorization on LinAlg and tensor ops.
+std::unique_ptr<OperationPass<func::FuncOp>> createGenericVectorizationPass();
+std::unique_ptr<OperationPass<func::FuncOp>>
+createGenericVectorizationPass(const GenericVectorizationPassOptions &options);
+
std::unique_ptr<OperationPass<func::FuncOp>>
createHoistStaticallyBoundAllocationsPass();
diff --git a/compiler/src/iree/compiler/Codegen/Common/Passes.td b/compiler/src/iree/compiler/Codegen/Common/Passes.td
index 3aa73da..ce1b356 100644
--- a/compiler/src/iree/compiler/Codegen/Common/Passes.td
+++ b/compiler/src/iree/compiler/Codegen/Common/Passes.td
@@ -212,6 +212,21 @@
let constructor = "mlir::iree_compiler::createFuseTensorPadWithConsumerPass()";
}
+def GenericVectorization :
+ Pass<"iree-codegen-generic-vectorization", "func::FuncOp"> {
+ let summary = "Pass to perform vectorization on tensor/linalg ops.";
+ let options = [
+ Option<"enableVectorMasking", "enable-vector-masking", "bool",/*default=*/"false",
+ "Enable vector masking during vectorization.">,
+ Option<"vectorizePadding", "vectorize-padding", "bool", /*default=*/"false",
+ "Rewrite all tensor.pad ops in the function to vector form.">,
+ Option<"vectorizeGatherAccesses", "vectorize-gather-accesses", "bool", /*default=*/"false",
+ "Enable vectorizaiton of operations that may generate vector.gather operations.">,
+ ];
+ let constructor =
+ "mlir::iree_compiler::createGenericVectorizationPass()";
+}
+
def HoistStaticallyBoundAllocations :
Pass<"iree-hoist-statically-bound-allocations", "func::FuncOp"> {
let summary = "Hoist statically bound alloca ops to the entry block of functions";
@@ -398,4 +413,4 @@
let constructor = "mlir::iree_compiler::createTypePropagationPass()";
}
-#endif // IREE_CODEGEN_COMMON_PASSES
\ No newline at end of file
+#endif // IREE_CODEGEN_COMMON_PASSES
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel
index 8d4b9ec..fad7490 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel
@@ -68,7 +68,6 @@
"LLVMCPUTileAndFuse.cpp",
"LLVMCPUUnfuseFMAOps.cpp",
"LLVMCPUVectorLowering.cpp",
- "LLVMCPUVectorization.cpp",
"Passes.cpp",
"TargetMLTransformInfo.cpp",
"TileSizeSelection.cpp",
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt
index fc68a04..ba776c0 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt
@@ -70,7 +70,6 @@
"LLVMCPUTileAndFuse.cpp"
"LLVMCPUUnfuseFMAOps.cpp"
"LLVMCPUVectorLowering.cpp"
- "LLVMCPUVectorization.cpp"
"Passes.cpp"
"TargetMLTransformInfo.cpp"
"TileSizeSelection.cpp"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp
index b2e0414..1caab24 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp
@@ -334,11 +334,11 @@
createLLVMCPUTilePass(tilingConfig.getVectorParallelLevel()));
nestedModulePM.addNestedPass<func::FuncOp>(createLLVMCPUPeelPass());
{
- LLVMCPUVectorizationPassOptions options;
+ GenericVectorizationPassOptions options;
options.enableVectorMasking = enableVectorMasking;
options.vectorizeGatherAccesses = true;
nestedModulePM.addNestedPass<func::FuncOp>(
- createLLVMCPUVectorizationPass(options));
+ createGenericVectorizationPass(options));
nestedModulePM.addNestedPass<func::FuncOp>(createCanonicalizerPass());
nestedModulePM.addNestedPass<func::FuncOp>(createCSEPass());
}
@@ -377,12 +377,12 @@
createLLVMCPUTensorPadPass(LLVMCPUTensorPadOption::ReductionDims));
{
- LLVMCPUVectorizationPassOptions options;
+ GenericVectorizationPassOptions options;
options.enableVectorMasking = enableVectorMasking;
options.vectorizePadding = true;
options.vectorizeGatherAccesses = true;
nestedModulePM.addNestedPass<func::FuncOp>(
- createLLVMCPUVectorizationPass(options));
+ createGenericVectorizationPass(options));
nestedModulePM.addNestedPass<func::FuncOp>(createCanonicalizerPass());
nestedModulePM.addNestedPass<func::FuncOp>(createCSEPass());
}
@@ -485,12 +485,12 @@
nestedModulePM.addNestedPass<func::FuncOp>(createCanonicalizerPass());
nestedModulePM.addNestedPass<func::FuncOp>(createCSEPass());
- LLVMCPUVectorizationPassOptions options;
+ GenericVectorizationPassOptions options;
options.enableVectorMasking = enableVectorMasking;
options.vectorizePadding = true;
options.vectorizeGatherAccesses = true;
nestedModulePM.addNestedPass<func::FuncOp>(
- createLLVMCPUVectorizationPass(options));
+ createGenericVectorizationPass(options));
nestedModulePM.addNestedPass<func::FuncOp>(createCanonicalizerPass());
nestedModulePM.addNestedPass<func::FuncOp>(createCSEPass());
}
@@ -546,12 +546,12 @@
{
nestedModulePM.addNestedPass<func::FuncOp>(createVectorizePadPass());
- LLVMCPUVectorizationPassOptions options;
+ GenericVectorizationPassOptions options;
options.enableVectorMasking = enableVectorMasking;
options.vectorizePadding = true;
options.vectorizeGatherAccesses = true;
nestedModulePM.addNestedPass<func::FuncOp>(
- createLLVMCPUVectorizationPass(options));
+ createGenericVectorizationPass(options));
nestedModulePM.addNestedPass<func::FuncOp>(createCanonicalizerPass());
nestedModulePM.addNestedPass<func::FuncOp>(createCSEPass());
}
@@ -598,7 +598,7 @@
nestedModulePM.addNestedPass<func::FuncOp>(createLLVMCPUTilePass(
static_cast<int64_t>(tilingConfig.getVectorReductionLevel())));
nestedModulePM.addNestedPass<func::FuncOp>(
- createLLVMCPUVectorizationPass());
+ createGenericVectorizationPass());
}
nestedModulePM.addNestedPass<func::FuncOp>(createCanonicalizerPass());
@@ -622,10 +622,10 @@
createDecomposePackUnPackOpsPass());
{
- LLVMCPUVectorizationPassOptions options;
+ GenericVectorizationPassOptions options;
options.vectorizePadding = true;
nestedModulePM.addNestedPass<func::FuncOp>(
- createLLVMCPUVectorizationPass(options));
+ createGenericVectorizationPass(options));
nestedModulePM.addNestedPass<func::FuncOp>(createCanonicalizerPass());
nestedModulePM.addNestedPass<func::FuncOp>(createCSEPass());
}
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h
index 926f296..fa64e96 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h
@@ -91,15 +91,6 @@
std::unique_ptr<OperationPass<func::FuncOp>> createLLVMCPUVectorLoweringPass(
const LLVMCPUVectorLoweringPassOptions &options);
-struct LLVMCPUVectorizationPassOptions {
- bool enableVectorMasking = false;
- bool vectorizePadding = false;
- bool vectorizeGatherAccesses = false;
-};
-std::unique_ptr<OperationPass<func::FuncOp>> createLLVMCPUVectorizationPass();
-std::unique_ptr<OperationPass<func::FuncOp>>
-createLLVMCPUVectorizationPass(const LLVMCPUVectorizationPassOptions &options);
-
/// A pass that converts certain vector.contract ops to custom kernels.
std::unique_ptr<OperationPass<func::FuncOp>>
createVectorContractCustomKernelsPass();
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td
index ca07c2a..6dd2edb 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td
@@ -152,21 +152,6 @@
let constructor = "mlir::iree_compiler::createLLVMCPUUnfuseFMAOpsPass()";
}
-def LLVMCPUVectorization :
- Pass<"iree-llvmcpu-vectorization", "func::FuncOp"> {
- let summary = "Pass to perform vectorization on tensor/linalg ops.";
- let options = [
- Option<"enableVectorMasking", "enable-vector-masking", "bool",/*default=*/"false",
- "Enable vector masking during vectorization.">,
- Option<"vectorizePadding", "vectorize-padding", "bool", /*default=*/"false",
- "Rewrite all tensor.pad ops in the function to vector form.">,
- Option<"vectorizeGatherAccesses", "vectorize-gather-accesses", "bool", /*default=*/"false",
- "Enable vectorizaiton of operations that may generate vector.gather operations.">,
- ];
- let constructor =
- "mlir::iree_compiler::createLLVMCPUVectorizationPass()";
-}
-
def LLVMCPUVectorLowering :
Pass<"iree-llvmcpu-vector-lowering", "func::FuncOp"> {
let summary = "Pass to lower Vector ops before conversion to LLVM.";
