NFC: Split `Passes.td` in `Codegen` into several `Passes.td` in individual codegen folders (#14230)
- Split Codegen/Passes.td into
- Codegen/Common/Passes.td
- Codegen/Common/GPU/Passes.td
- Codegen/LLVMCPU/Passes.td
- Codegen/LLVMGPU/Passes.td
- Codegen/SPIRV/Passes.td
- Codegen/VMVX/Passes.td
- Codegen/WGSL/Passes.td
- Delete Codegen/Passes.td
- Rename unnecessarily verbose header file
- Common/CommonPasses.h -> Common/Passes.h
- Common/GPU/CommonGPUPasses.h -> Common/GPU/Passes.h
- LLVMCPU/LLVMCPUPasses.h -> LLVMCPU/Passes.h
- LLVMGPU/LLVMGPUPasses.h -> LLVMGPU/Passes.h
- SPIRV/SPIRVPasses.h -> SPIRV/Passes.h
- VMVX/VMVXPasses.h -> VMVX/Passes.h
- WGSL/WGSLPasses.h -> WGSL/Passes.h
diff --git a/compiler/plugins/target/CUDA/CUDATarget.cpp b/compiler/plugins/target/CUDA/CUDATarget.cpp
index 1f46d35..0e9ad1e 100644
--- a/compiler/plugins/target/CUDA/CUDATarget.cpp
+++ b/compiler/plugins/target/CUDA/CUDATarget.cpp
@@ -6,7 +6,7 @@
#include "./LLVMPasses.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Dialect/HAL/Target/LLVMLinkerUtils.h"
#include "iree/compiler/Dialect/HAL/Target/TargetRegistry.h"
#include "iree/compiler/PluginAPI/Client.h"
diff --git a/compiler/src/iree/compiler/Codegen/BUILD.bazel b/compiler/src/iree/compiler/Codegen/BUILD.bazel
index 34ab2a3..5d6a3fc 100644
--- a/compiler/src/iree/compiler/Codegen/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/BUILD.bazel
@@ -4,7 +4,7 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library", "iree_gentbl_cc_library")
+load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library")
package(
default_visibility = ["//visibility:public"],
@@ -12,49 +12,16 @@
licenses = ["notice"], # Apache 2.0
)
-iree_gentbl_cc_library(
- name = "PassesIncGen",
- tbl_outs = [
- (
- ["--gen-pass-decls"],
- "Passes.h.inc",
- ),
- ],
- tblgen = "@llvm-project//mlir:mlir-tblgen",
- td_file = "Passes.td",
- deps = ["@llvm-project//mlir:PassBaseTdFiles"],
-)
-
-iree_compiler_cc_library(
- name = "PassHeaders",
- hdrs = [
- "PassDetail.h",
- "Passes.h",
- "Passes.h.inc",
- ],
- deps = [
- ":PassesIncGen",
- "//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
- "//compiler/src/iree/compiler/Dialect/HAL/IR",
- "//compiler/src/iree/compiler/Utils",
- "@llvm-project//mlir:LinalgTransforms",
- "@llvm-project//mlir:MemRefDialect",
- "@llvm-project//mlir:NVGPUDialect",
- "@llvm-project//mlir:Pass",
- "@llvm-project//mlir:Transforms",
- ],
-)
-
iree_compiler_cc_library(
name = "Codegen",
srcs = [
"Passes.cpp",
],
+ hdrs = [
+ "Passes.h",
+ ],
deps = [
- ":PassHeaders",
- ":PassesIncGen",
"//compiler/src/iree/compiler/Codegen/Common",
- "//compiler/src/iree/compiler/Codegen/Common:CommonPasses",
"//compiler/src/iree/compiler/Codegen/Common/GPU:CommonGPUPasses",
"//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
"//compiler/src/iree/compiler/Codegen/LLVMCPU",
diff --git a/compiler/src/iree/compiler/Codegen/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/CMakeLists.txt
index 55a90cf..40dc173 100644
--- a/compiler/src/iree/compiler/Codegen/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/CMakeLists.txt
@@ -10,47 +10,17 @@
iree_add_all_subdirs()
-iree_tablegen_library(
- NAME
- PassesIncGen
- TD_FILE
- "Passes.td"
- OUTS
- --gen-pass-decls Passes.h.inc
-)
-
-iree_cc_library(
- NAME
- PassHeaders
- HDRS
- "PassDetail.h"
- "Passes.h"
- "Passes.h.inc"
- DEPS
- ::PassesIncGen
- MLIRLinalgTransforms
- MLIRMemRefDialect
- MLIRNVGPUDialect
- MLIRPass
- MLIRTransforms
- iree::compiler::Codegen::Dialect::IREECodegenDialect
- iree::compiler::Dialect::HAL::IR
- iree::compiler::Utils
- PUBLIC
-)
-
iree_cc_library(
NAME
Codegen
+ HDRS
+ "Passes.h"
SRCS
"Passes.cpp"
DEPS
- ::PassHeaders
- ::PassesIncGen
IREELinalgExtPasses
MLIRPass
iree::compiler::Codegen::Common
- iree::compiler::Codegen::Common::CommonPasses
iree::compiler::Codegen::Common::GPU::CommonGPUPasses
iree::compiler::Codegen::Dialect::IREECodegenDialect
iree::compiler::Codegen::LLVMCPU
@@ -62,8 +32,3 @@
)
### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
-# TODO: For some reason, these dependencies are not being added automatically.
-add_dependencies(
- iree_compiler_Codegen_PassHeaders
- iree_compiler_Codegen_PassesIncGen
-)
diff --git a/compiler/src/iree/compiler/Codegen/Common/BUILD.bazel b/compiler/src/iree/compiler/Codegen/Common/BUILD.bazel
index 6c2b049..69c0317 100644
--- a/compiler/src/iree/compiler/Codegen/Common/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/Common/BUILD.bazel
@@ -13,6 +13,38 @@
)
iree_gentbl_cc_library(
+ name = "PassesIncGen",
+ tbl_outs = [
+ (
+ ["--gen-pass-decls"],
+ "Passes.h.inc",
+ ),
+ ],
+ tblgen = "@llvm-project//mlir:mlir-tblgen",
+ td_file = "Passes.td",
+ deps = ["@llvm-project//mlir:PassBaseTdFiles"],
+)
+
+iree_compiler_cc_library(
+ name = "PassHeaders",
+ hdrs = [
+ "PassDetail.h",
+ "Passes.h",
+ "Passes.h.inc",
+ ],
+ deps = [
+ ":PassesIncGen",
+ "//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
+ "//compiler/src/iree/compiler/Dialect/HAL/IR",
+ "//compiler/src/iree/compiler/Utils",
+ "@llvm-project//mlir:LinalgTransforms",
+ "@llvm-project//mlir:MemRefDialect",
+ "@llvm-project//mlir:Pass",
+ "@llvm-project//mlir:Transforms",
+ ],
+)
+
+iree_gentbl_cc_library(
name = "FoldTensorExtractOpIncGen",
tbl_outs = [
(
@@ -46,6 +78,8 @@
"TransformDialectInterpreterPass.cpp",
],
deps = [
+ ":PassHeaders",
+ ":PassesIncGen",
# Dialects
"//compiler/src/iree/compiler/Dialect/Flow/IR",
"//llvm-external-projects/iree-dialects:IREELinalgExtDialect",
@@ -83,7 +117,6 @@
"@llvm-project//mlir:TensorTransforms",
"@llvm-project//mlir:VectorTransforms",
# Other Stuff
- "//compiler/src/iree/compiler/Codegen:PassHeaders",
"//compiler/src/iree/compiler/Utils",
"@llvm-project//llvm:Support",
"@llvm-project//mlir:Support",
@@ -108,7 +141,7 @@
)
iree_compiler_cc_library(
- name = "CommonPasses",
+ name = "Common",
srcs = [
"BubbleUpOrdinalOps.cpp",
"BufferizationAnalysis.cpp",
@@ -119,9 +152,13 @@
"ConvertBf16ToUInt16Buffers.cpp",
"ConvertToDestinationPassingStylePass.cpp",
"DecomposeAffineOpsPass.cpp",
+ "DecomposeConvolutionToLowerDimOps.cpp",
+ "DecomposeLinalgGeneric.cpp",
"DecomposePackUnPackOps.cpp",
"EraseHALDescriptorTypeFromMemRef.cpp",
+ "ExtractAddressComputation.cpp",
"FlattenMemRefSubspanPass.cpp",
+ "FoldAffineMinInDistributedLoops.cpp",
"FoldTensorExtractOpPass.cpp",
"ForOpCanonicalizationPass.cpp",
"FuseTensorPadWithConsumer.cpp",
@@ -131,26 +168,33 @@
"InstrumentMemoryAccesses.cpp",
"LowerUKernelsToCalls.cpp",
"MaterializeEncodingIntoPackUnPack.cpp",
+ "MemrefCopyToLinalg.cpp",
"OptimizeVectorTransferPass.cpp",
+ "PadDynamicAlloc.cpp",
+ "Passes.cpp",
"PolynomialApproximationPass.cpp",
"RematerializeParallelOps.cpp",
+ "RemoveTrivialLoops.cpp",
"SplitFullPartialTransferPass.cpp",
"TestExecutablePreprocessing.cpp",
"TestPartitionableLoopsInterface.cpp",
"TileAndDistributeToWorkgroupsPass.cpp",
"TileDispatchUsingInterface.cpp",
"TypePropagationPass.cpp",
+ "UserConfig.cpp",
"VectorizePad.cpp",
- "WorkgroupSpecializationPass.cpp",
],
hdrs = [
"BufferizationAnalysis.h",
- "CommonPasses.h",
"EncodingInfo.h",
+ "ExtractAddressComputation.h",
+ "Passes.h",
"Transforms.h",
+ "UserConfig.h",
],
deps = [
- "//compiler/src/iree/compiler/Codegen:PassHeaders",
+ ":PassHeaders",
+ ":PassesIncGen",
"//compiler/src/iree/compiler/Codegen/Common:FoldTensorExtractOpIncGen",
"//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
"//compiler/src/iree/compiler/Codegen/Interfaces:BufferizationInterfaces",
@@ -169,6 +213,7 @@
"//llvm-external-projects/iree-dialects:IREELinalgExtTransforms",
"//runtime/src/iree/builtins/ukernel:exported_bits",
"@llvm-project//llvm:Support",
+ "@llvm-project//mlir:AffineAnalysis",
"@llvm-project//mlir:AffineDialect",
"@llvm-project//mlir:AffineTransforms",
"@llvm-project//mlir:AffineUtils",
@@ -204,82 +249,8 @@
"@llvm-project//mlir:TilingInterface",
"@llvm-project//mlir:TransformDialect",
"@llvm-project//mlir:Transforms",
- "@llvm-project//mlir:VectorDialect",
- "@llvm-project//mlir:VectorTransforms",
- "@llvm-project//mlir:ViewLikeInterface",
- ],
-)
-
-iree_compiler_cc_library(
- name = "Common",
- srcs = [
- "DecomposeConvolutionToLowerDimOps.cpp",
- "DecomposeLinalgGeneric.cpp",
- "ExtractAddressComputation.cpp",
- "FoldAffineMinInDistributedLoops.cpp",
- "MemrefCopyToLinalg.cpp",
- "PadDynamicAlloc.cpp",
- "RemoveTrivialLoops.cpp",
- "UserConfig.cpp",
- ],
- hdrs = [
- "ExtractAddressComputation.h",
- "UserConfig.h",
- ],
- deps = [
- ":CommonPasses",
- ":TransformDialectInterpreterPass",
- "//compiler/src/iree/compiler/Codegen:PassHeaders",
- "//compiler/src/iree/compiler/Codegen/Common:FoldTensorExtractOpIncGen",
- "//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
- "//compiler/src/iree/compiler/Codegen/Interfaces:BufferizationInterfaces",
- "//compiler/src/iree/compiler/Codegen/Interfaces:PartitionableLoopsInterface",
- "//compiler/src/iree/compiler/Codegen/Transforms",
- "//compiler/src/iree/compiler/Codegen/Utils",
- "//compiler/src/iree/compiler/Dialect/Flow/IR",
- "//compiler/src/iree/compiler/Dialect/HAL/IR",
- "//compiler/src/iree/compiler/Dialect/Util/IR",
- "//compiler/src/iree/compiler/Utils",
- "//llvm-external-projects/iree-dialects:IREELinalgExtDialect",
- "//llvm-external-projects/iree-dialects:IREELinalgExtPasses",
- "//llvm-external-projects/iree-dialects:IREELinalgExtTransforms",
- "@llvm-project//llvm:Support",
- "@llvm-project//mlir:AffineAnalysis",
- "@llvm-project//mlir:AffineDialect",
- "@llvm-project//mlir:AffineUtils",
- "@llvm-project//mlir:Analysis",
- "@llvm-project//mlir:ArithDialect",
- "@llvm-project//mlir:ArithTransforms",
- "@llvm-project//mlir:ArithUtils",
- "@llvm-project//mlir:BufferizationDialect",
- "@llvm-project//mlir:BufferizationTransforms",
- "@llvm-project//mlir:DialectUtils",
- "@llvm-project//mlir:FuncDialect",
- "@llvm-project//mlir:GPUDialect",
- "@llvm-project//mlir:IR",
- "@llvm-project//mlir:LLVMCommonConversion",
- "@llvm-project//mlir:LLVMDialect",
- "@llvm-project//mlir:LinalgDialect",
- "@llvm-project//mlir:LinalgTransforms",
- "@llvm-project//mlir:MathDialect",
- "@llvm-project//mlir:MathTransforms",
- "@llvm-project//mlir:MemRefDialect",
- "@llvm-project//mlir:MemRefTransforms",
- "@llvm-project//mlir:Pass",
- "@llvm-project//mlir:SCFDialect",
- "@llvm-project//mlir:SCFToControlFlow",
- "@llvm-project//mlir:SCFTransforms",
- "@llvm-project//mlir:SCFUtils",
- "@llvm-project//mlir:SideEffectInterfaces",
- "@llvm-project//mlir:Support",
- "@llvm-project//mlir:TensorDialect",
- "@llvm-project//mlir:TensorTransforms",
- "@llvm-project//mlir:TilingInterface",
- "@llvm-project//mlir:TransformDialect",
- "@llvm-project//mlir:Transforms",
"@llvm-project//mlir:ValueBoundsOpInterface",
"@llvm-project//mlir:VectorDialect",
- "@llvm-project//mlir:VectorToSCF",
"@llvm-project//mlir:VectorTransforms",
"@llvm-project//mlir:ViewLikeInterface",
],
diff --git a/compiler/src/iree/compiler/Codegen/Common/BubbleUpOrdinalOps.cpp b/compiler/src/iree/compiler/Codegen/Common/BubbleUpOrdinalOps.cpp
index 1f03305..4f6f743 100644
--- a/compiler/src/iree/compiler/Codegen/Common/BubbleUpOrdinalOps.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/BubbleUpOrdinalOps.cpp
@@ -11,8 +11,8 @@
//
//===---------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/BufferizeCopyOnlyDispatchesPass.cpp b/compiler/src/iree/compiler/Codegen/Common/BufferizeCopyOnlyDispatchesPass.cpp
index 007eaee..a8e0a70 100644
--- a/compiler/src/iree/compiler/Codegen/Common/BufferizeCopyOnlyDispatchesPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/BufferizeCopyOnlyDispatchesPass.cpp
@@ -11,8 +11,8 @@
//
//===----------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/Flow/IR/FlowDialect.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Common/CMakeLists.txt
index 6892350..532e535 100644
--- a/compiler/src/iree/compiler/Codegen/Common/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/Common/CMakeLists.txt
@@ -12,6 +12,34 @@
iree_tablegen_library(
NAME
+ PassesIncGen
+ TD_FILE
+ "Passes.td"
+ OUTS
+ --gen-pass-decls Passes.h.inc
+)
+
+iree_cc_library(
+ NAME
+ PassHeaders
+ HDRS
+ "PassDetail.h"
+ "Passes.h"
+ "Passes.h.inc"
+ DEPS
+ ::PassesIncGen
+ MLIRLinalgTransforms
+ MLIRMemRefDialect
+ MLIRPass
+ MLIRTransforms
+ iree::compiler::Codegen::Dialect::IREECodegenDialect
+ iree::compiler::Dialect::HAL::IR
+ iree::compiler::Utils
+ PUBLIC
+)
+
+iree_tablegen_library(
+ NAME
FoldTensorExtractOpIncGen
TD_FILE
"FoldTensorExtractOp.td"
@@ -25,6 +53,8 @@
SRCS
"TransformDialectInterpreterPass.cpp"
DEPS
+ ::PassHeaders
+ ::PassesIncGen
IREEDialectsTransforms
IREELinalgExtDialect
IREELinalgExtTransformOps
@@ -70,7 +100,6 @@
iree::compiler::Codegen::Common::TransformExtensions::CommonExtensions
iree::compiler::Codegen::LLVMCPU::TransformExtensions::LLVMCPUExtensions
iree::compiler::Codegen::LLVMGPU::TransformExtensions::LLVMGPUExtensions
- iree::compiler::Codegen::PassHeaders
iree::compiler::Codegen::TransformStrategies::Common::TransformStrategies
iree::compiler::Dialect::Flow::IR
iree::compiler::Dialect::Flow::TransformExtensions::FlowExtensions
@@ -80,12 +109,14 @@
iree_cc_library(
NAME
- CommonPasses
+ Common
HDRS
"BufferizationAnalysis.h"
- "CommonPasses.h"
"EncodingInfo.h"
+ "ExtractAddressComputation.h"
+ "Passes.h"
"Transforms.h"
+ "UserConfig.h"
SRCS
"BubbleUpOrdinalOps.cpp"
"BufferizationAnalysis.cpp"
@@ -96,9 +127,13 @@
"ConvertBf16ToUInt16Buffers.cpp"
"ConvertToDestinationPassingStylePass.cpp"
"DecomposeAffineOpsPass.cpp"
+ "DecomposeConvolutionToLowerDimOps.cpp"
+ "DecomposeLinalgGeneric.cpp"
"DecomposePackUnPackOps.cpp"
"EraseHALDescriptorTypeFromMemRef.cpp"
+ "ExtractAddressComputation.cpp"
"FlattenMemRefSubspanPass.cpp"
+ "FoldAffineMinInDistributedLoops.cpp"
"FoldTensorExtractOpPass.cpp"
"ForOpCanonicalizationPass.cpp"
"FuseTensorPadWithConsumer.cpp"
@@ -108,22 +143,29 @@
"InstrumentMemoryAccesses.cpp"
"LowerUKernelsToCalls.cpp"
"MaterializeEncodingIntoPackUnPack.cpp"
+ "MemrefCopyToLinalg.cpp"
"OptimizeVectorTransferPass.cpp"
+ "PadDynamicAlloc.cpp"
+ "Passes.cpp"
"PolynomialApproximationPass.cpp"
"RematerializeParallelOps.cpp"
+ "RemoveTrivialLoops.cpp"
"SplitFullPartialTransferPass.cpp"
"TestExecutablePreprocessing.cpp"
"TestPartitionableLoopsInterface.cpp"
"TileAndDistributeToWorkgroupsPass.cpp"
"TileDispatchUsingInterface.cpp"
"TypePropagationPass.cpp"
+ "UserConfig.cpp"
"VectorizePad.cpp"
- "WorkgroupSpecializationPass.cpp"
DEPS
+ ::PassHeaders
+ ::PassesIncGen
IREELinalgExtDialect
IREELinalgExtPasses
IREELinalgExtTransforms
LLVMSupport
+ MLIRAffineAnalysis
MLIRAffineDialect
MLIRAffineTransforms
MLIRAffineUtils
@@ -158,6 +200,7 @@
MLIRTilingInterface
MLIRTransformDialect
MLIRTransforms
+ MLIRValueBoundsOpInterface
MLIRVectorDialect
MLIRVectorTransforms
MLIRViewLikeInterface
@@ -167,7 +210,6 @@
iree::compiler::Codegen::Interfaces::BufferizationInterfaces
iree::compiler::Codegen::Interfaces::PartitionableLoopsInterface
iree::compiler::Codegen::Interfaces::UKernelOpInterface
- iree::compiler::Codegen::PassHeaders
iree::compiler::Codegen::Transforms
iree::compiler::Codegen::Utils
iree::compiler::Dialect::Flow::IR
@@ -179,77 +221,4 @@
PUBLIC
)
-iree_cc_library(
- NAME
- Common
- HDRS
- "ExtractAddressComputation.h"
- "UserConfig.h"
- SRCS
- "DecomposeConvolutionToLowerDimOps.cpp"
- "DecomposeLinalgGeneric.cpp"
- "ExtractAddressComputation.cpp"
- "FoldAffineMinInDistributedLoops.cpp"
- "MemrefCopyToLinalg.cpp"
- "PadDynamicAlloc.cpp"
- "RemoveTrivialLoops.cpp"
- "UserConfig.cpp"
- DEPS
- ::CommonPasses
- ::TransformDialectInterpreterPass
- IREELinalgExtDialect
- IREELinalgExtPasses
- IREELinalgExtTransforms
- LLVMSupport
- MLIRAffineAnalysis
- MLIRAffineDialect
- MLIRAffineUtils
- MLIRAnalysis
- MLIRArithDialect
- MLIRArithTransforms
- MLIRArithUtils
- MLIRBufferizationDialect
- MLIRBufferizationTransforms
- MLIRFuncDialect
- MLIRGPUDialect
- MLIRIR
- MLIRLLVMCommonConversion
- MLIRLLVMDialect
- MLIRLinalgDialect
- MLIRLinalgTransforms
- MLIRMathDialect
- MLIRMathTransforms
- MLIRMemRefDialect
- MLIRMemRefTransforms
- MLIRPass
- MLIRSCFDialect
- MLIRSCFToControlFlow
- MLIRSCFTransforms
- MLIRSCFUtils
- MLIRSideEffectInterfaces
- MLIRSupport
- MLIRTensorDialect
- MLIRTensorTransforms
- MLIRTilingInterface
- MLIRTransformDialect
- MLIRTransforms
- MLIRValueBoundsOpInterface
- MLIRVectorDialect
- MLIRVectorToSCF
- MLIRVectorTransforms
- MLIRViewLikeInterface
- iree::compiler::Codegen::Common::FoldTensorExtractOpIncGen
- iree::compiler::Codegen::Dialect::IREECodegenDialect
- iree::compiler::Codegen::Interfaces::BufferizationInterfaces
- iree::compiler::Codegen::Interfaces::PartitionableLoopsInterface
- iree::compiler::Codegen::PassHeaders
- iree::compiler::Codegen::Transforms
- iree::compiler::Codegen::Utils
- iree::compiler::Dialect::Flow::IR
- iree::compiler::Dialect::HAL::IR
- iree::compiler::Dialect::Util::IR
- iree::compiler::Utils
- PUBLIC
-)
-
### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
diff --git a/compiler/src/iree/compiler/Codegen/Common/CleanupBufferAllocViewPass.cpp b/compiler/src/iree/compiler/Codegen/Common/CleanupBufferAllocViewPass.cpp
index ddb90d8..5a3363a 100644
--- a/compiler/src/iree/compiler/Codegen/Common/CleanupBufferAllocViewPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/CleanupBufferAllocViewPass.cpp
@@ -12,8 +12,8 @@
//
//===----------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/ConcretizePadResultShape.cpp b/compiler/src/iree/compiler/Codegen/Common/ConcretizePadResultShape.cpp
index a1a1d57..98556fd 100644
--- a/compiler/src/iree/compiler/Codegen/Common/ConcretizePadResultShape.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/ConcretizePadResultShape.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/ConvertBf16ArithToF32.cpp b/compiler/src/iree/compiler/Codegen/Common/ConvertBf16ArithToF32.cpp
index f620ad9..29e9f4a 100644
--- a/compiler/src/iree/compiler/Codegen/Common/ConvertBf16ArithToF32.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/ConvertBf16ArithToF32.cpp
@@ -14,8 +14,8 @@
#include <memory>
#include <utility>
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "iree/compiler/Utils/ConversionUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/ConvertBf16ToUInt16Buffers.cpp b/compiler/src/iree/compiler/Codegen/Common/ConvertBf16ToUInt16Buffers.cpp
index 0219f17..1df9f53 100644
--- a/compiler/src/iree/compiler/Codegen/Common/ConvertBf16ToUInt16Buffers.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/ConvertBf16ToUInt16Buffers.cpp
@@ -11,8 +11,8 @@
//
//===----------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "iree/compiler/Utils/ConversionUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/ConvertToDestinationPassingStylePass.cpp b/compiler/src/iree/compiler/Codegen/Common/ConvertToDestinationPassingStylePass.cpp
index 938fdc8..6bf71da 100644
--- a/compiler/src/iree/compiler/Codegen/Common/ConvertToDestinationPassingStylePass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/ConvertToDestinationPassingStylePass.cpp
@@ -14,8 +14,8 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtOps.h"
#include "iree/compiler/Codegen/Common/BufferizationAnalysis.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/DecomposeAffineOpsPass.cpp b/compiler/src/iree/compiler/Codegen/Common/DecomposeAffineOpsPass.cpp
index 42514d3..a7f7fe0 100644
--- a/compiler/src/iree/compiler/Codegen/Common/DecomposeAffineOpsPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/DecomposeAffineOpsPass.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Affine/Transforms/Transforms.h"
namespace mlir {
diff --git a/compiler/src/iree/compiler/Codegen/Common/DecomposeConvolutionToLowerDimOps.cpp b/compiler/src/iree/compiler/Codegen/Common/DecomposeConvolutionToLowerDimOps.cpp
index 911969a..a46fff8 100644
--- a/compiler/src/iree/compiler/Codegen/Common/DecomposeConvolutionToLowerDimOps.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/DecomposeConvolutionToLowerDimOps.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/DecomposeLinalgGeneric.cpp b/compiler/src/iree/compiler/Codegen/Common/DecomposeLinalgGeneric.cpp
index a22964a..9ecd43f 100644
--- a/compiler/src/iree/compiler/Codegen/Common/DecomposeLinalgGeneric.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/DecomposeLinalgGeneric.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/DecomposePackUnPackOps.cpp b/compiler/src/iree/compiler/Codegen/Common/DecomposePackUnPackOps.cpp
index 2ce5edf..01852af 100644
--- a/compiler/src/iree/compiler/Codegen/Common/DecomposePackUnPackOps.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/DecomposePackUnPackOps.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/EraseHALDescriptorTypeFromMemRef.cpp b/compiler/src/iree/compiler/Codegen/Common/EraseHALDescriptorTypeFromMemRef.cpp
index 1ac61e5..d005dbc 100644
--- a/compiler/src/iree/compiler/Codegen/Common/EraseHALDescriptorTypeFromMemRef.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/EraseHALDescriptorTypeFromMemRef.cpp
@@ -13,8 +13,8 @@
#include <memory>
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Dialect/HAL/IR/HALTypes.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/IR/Attributes.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/ExtractAddressComputation.cpp b/compiler/src/iree/compiler/Codegen/Common/ExtractAddressComputation.cpp
index aa7ced7..f3704e9 100644
--- a/compiler/src/iree/compiler/Codegen/Common/ExtractAddressComputation.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/ExtractAddressComputation.cpp
@@ -6,8 +6,8 @@
#include "iree/compiler/Codegen/Common/ExtractAddressComputation.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/FlattenMemRefSubspanPass.cpp b/compiler/src/iree/compiler/Codegen/Common/FlattenMemRefSubspanPass.cpp
index db1f8d6..f5c6856 100644
--- a/compiler/src/iree/compiler/Codegen/Common/FlattenMemRefSubspanPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/FlattenMemRefSubspanPass.cpp
@@ -33,9 +33,9 @@
#include <memory>
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Dialect/UKernelOps.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/FoldAffineMinInDistributedLoops.cpp b/compiler/src/iree/compiler/Codegen/Common/FoldAffineMinInDistributedLoops.cpp
index c26a599..ce744b4 100644
--- a/compiler/src/iree/compiler/Codegen/Common/FoldAffineMinInDistributedLoops.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/FoldAffineMinInDistributedLoops.cpp
@@ -14,8 +14,8 @@
//
//===----------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "llvm/ADT/STLExtras.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/FoldTensorExtractOpPass.cpp b/compiler/src/iree/compiler/Codegen/Common/FoldTensorExtractOpPass.cpp
index 23e51a9..f51f2eb 100644
--- a/compiler/src/iree/compiler/Codegen/Common/FoldTensorExtractOpPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/FoldTensorExtractOpPass.cpp
@@ -3,8 +3,8 @@
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/ForOpCanonicalizationPass.cpp b/compiler/src/iree/compiler/Codegen/Common/ForOpCanonicalizationPass.cpp
index 37d4172..31d042a 100644
--- a/compiler/src/iree/compiler/Codegen/Common/ForOpCanonicalizationPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/ForOpCanonicalizationPass.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/Builders.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/FuseTensorPadWithConsumer.cpp b/compiler/src/iree/compiler/Codegen/Common/FuseTensorPadWithConsumer.cpp
index 9f9b44f..e022d18 100644
--- a/compiler/src/iree/compiler/Codegen/Common/FuseTensorPadWithConsumer.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/FuseTensorPadWithConsumer.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel
index 5a7c5d5..58d193d 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/BUILD.bazel
@@ -4,7 +4,7 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library")
+load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library", "iree_gentbl_cc_library")
package(
default_visibility = ["//visibility:public"],
@@ -12,6 +12,39 @@
licenses = ["notice"], # Apache 2.0
)
+iree_gentbl_cc_library(
+ name = "PassesIncGen",
+ tbl_outs = [
+ (
+ ["--gen-pass-decls"],
+ "Passes.h.inc",
+ ),
+ ],
+ tblgen = "@llvm-project//mlir:mlir-tblgen",
+ td_file = "Passes.td",
+ deps = ["@llvm-project//mlir:PassBaseTdFiles"],
+)
+
+iree_compiler_cc_library(
+ name = "PassHeaders",
+ hdrs = [
+ "PassDetail.h",
+ "Passes.h",
+ "Passes.h.inc",
+ ],
+ deps = [
+ ":PassesIncGen",
+ "//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
+ "//compiler/src/iree/compiler/Dialect/HAL/IR",
+ "//compiler/src/iree/compiler/Utils",
+ "@llvm-project//mlir:LinalgTransforms",
+ "@llvm-project//mlir:MemRefDialect",
+ "@llvm-project//mlir:NVGPUDialect",
+ "@llvm-project//mlir:Pass",
+ "@llvm-project//mlir:Transforms",
+ ],
+)
+
iree_compiler_cc_library(
name = "CommonGPUPasses",
srcs = [
@@ -26,15 +59,18 @@
"GPUTensorTile.cpp",
"GPUTileReduction.cpp",
"GPUVectorization.cpp",
+ "Passes.cpp",
"VectorReductionToGPU.cpp",
"WorkGroupSwizzle.cpp",
+ "WorkgroupSpecializationPass.cpp",
],
hdrs = [
- "CommonGPUPasses.h",
"GPUPatterns.h",
+ "Passes.h",
],
deps = [
- "//compiler/src/iree/compiler/Codegen:PassHeaders",
+ ":PassHeaders",
+ ":PassesIncGen",
"//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
"//compiler/src/iree/compiler/Codegen/Transforms",
"//compiler/src/iree/compiler/Codegen/Utils",
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt
index aad3fdd..3c775c9 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/CMakeLists.txt
@@ -10,12 +10,41 @@
iree_add_all_subdirs()
+iree_tablegen_library(
+ NAME
+ PassesIncGen
+ TD_FILE
+ "Passes.td"
+ OUTS
+ --gen-pass-decls Passes.h.inc
+)
+
+iree_cc_library(
+ NAME
+ PassHeaders
+ HDRS
+ "PassDetail.h"
+ "Passes.h"
+ "Passes.h.inc"
+ DEPS
+ ::PassesIncGen
+ MLIRLinalgTransforms
+ MLIRMemRefDialect
+ MLIRNVGPUDialect
+ MLIRPass
+ MLIRTransforms
+ iree::compiler::Codegen::Dialect::IREECodegenDialect
+ iree::compiler::Dialect::HAL::IR
+ iree::compiler::Utils
+ PUBLIC
+)
+
iree_cc_library(
NAME
CommonGPUPasses
HDRS
- "CommonGPUPasses.h"
"GPUPatterns.h"
+ "Passes.h"
SRCS
"GPUCheckResourceUsage.cpp"
"GPUDistribute.cpp"
@@ -28,9 +57,13 @@
"GPUTensorTile.cpp"
"GPUTileReduction.cpp"
"GPUVectorization.cpp"
+ "Passes.cpp"
"VectorReductionToGPU.cpp"
"WorkGroupSwizzle.cpp"
+ "WorkgroupSpecializationPass.cpp"
DEPS
+ ::PassHeaders
+ ::PassesIncGen
IREELinalgExtDialect
IREELinalgExtTransforms
LLVMSupport
@@ -64,7 +97,6 @@
MLIRVectorToSCF
MLIRVectorTransforms
iree::compiler::Codegen::Dialect::IREECodegenDialect
- iree::compiler::Codegen::PassHeaders
iree::compiler::Codegen::Transforms
iree::compiler::Codegen::Utils
iree::compiler::Dialect::HAL::IR
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUCheckResourceUsage.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUCheckResourceUsage.cpp
index af419b5..dd0f3ad 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUCheckResourceUsage.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUCheckResourceUsage.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "llvm/Support/CommandLine.h"
#include "mlir/Conversion/LLVMCommon/LoweringOptions.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUDistribute.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUDistribute.cpp
index 26947a3..837d028 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUDistribute.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUDistribute.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUDistributeSharedMemoryCopy.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUDistributeSharedMemoryCopy.cpp
index f6ddd91..dd0378b 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUDistributeSharedMemoryCopy.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUDistributeSharedMemoryCopy.cpp
@@ -9,9 +9,9 @@
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUMultiBuffering.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUMultiBuffering.cpp
index 032cbf4..e4a7e93 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUMultiBuffering.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUMultiBuffering.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUPipelining.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUPipelining.cpp
index 2cecafc..2f8ada9 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUPipelining.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUPipelining.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Dialect/HAL/IR/HALTypes.h"
#include "llvm/Support/Debug.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUReduceBankConflicts.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUReduceBankConflicts.cpp
index 4c3d091..ce87fba 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUReduceBankConflicts.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUReduceBankConflicts.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "mlir/Dialect/GPU/Transforms/Passes.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTensorAlloc.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTensorAlloc.cpp
index 16072f5..bb4cfee 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTensorAlloc.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTensorAlloc.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/LinalgOpInfo.h"
#include "llvm/Support/Debug.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTensorTile.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTensorTile.cpp
index 4e0b99c..73d9f61 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTensorTile.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTensorTile.cpp
@@ -8,9 +8,9 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtOps.h"
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTileReduction.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTileReduction.cpp
index 6ca051d..87d22fe 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTileReduction.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUTileReduction.cpp
@@ -4,9 +4,9 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUVectorization.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUVectorization.cpp
index 8f1a4fd..cb70a8b 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/GPUVectorization.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/GPUVectorization.cpp
@@ -6,8 +6,8 @@
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
diff --git a/compiler/src/iree/compiler/Codegen/PassDetail.h b/compiler/src/iree/compiler/Codegen/Common/GPU/PassDetail.h
similarity index 68%
copy from compiler/src/iree/compiler/Codegen/PassDetail.h
copy to compiler/src/iree/compiler/Codegen/Common/GPU/PassDetail.h
index fea75f0..f72494a 100644
--- a/compiler/src/iree/compiler/Codegen/PassDetail.h
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/PassDetail.h
@@ -1,11 +1,11 @@
-// Copyright 2021 The IREE Authors
+// Copyright 2023 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#ifndef IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
-#define IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#ifndef IREE_COMPILER_CODEGEN_COMMON_GPU_PASS_DETAIL_H_
+#define IREE_COMPILER_CODEGEN_COMMON_GPU_PASS_DETAIL_H_
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
@@ -18,9 +18,9 @@
namespace iree_compiler {
#define GEN_PASS_CLASSES
-#include "iree/compiler/Codegen/Passes.h.inc"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h.inc"
} // namespace iree_compiler
} // namespace mlir
-#endif // IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#endif // IREE_COMPILER_CODEGEN_COMMON_GPU_PASS_DETAIL_H_
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/Passes.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/Passes.cpp
new file mode 100644
index 0000000..48fad71
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/Passes.cpp
@@ -0,0 +1,23 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
+#include "mlir/Pass/PassManager.h"
+
+namespace mlir {
+namespace iree_compiler {
+
+namespace {
+#define GEN_PASS_REGISTRATION
+#include "iree/compiler/Codegen/Common/GPU/Passes.h.inc"
+} // namespace
+
+void registerCodegenCommonGPUPasses() {
+ // Generated.
+ registerPasses();
+}
+} // namespace iree_compiler
+} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h b/compiler/src/iree/compiler/Codegen/Common/GPU/Passes.h
similarity index 93%
rename from compiler/src/iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h
rename to compiler/src/iree/compiler/Codegen/Common/GPU/Passes.h
index 7c6f4d0..18ca8a1 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/Passes.h
@@ -5,8 +5,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#ifndef IREE_COMPILER_CODEGEN_COMMON_GPU_COMMONGPUASSES_H_
-#define IREE_COMPILER_CODEGEN_COMMON_GPU_COMMONGPUASSES_H_
+#ifndef IREE_COMPILER_CODEGEN_COMMON_GPU_PASSES_H_
+#define IREE_COMPILER_CODEGEN_COMMON_GPU_PASSES_H_
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
@@ -117,11 +117,18 @@
createConvertVectorReductionToGPUPass(
std::function<int(func::FuncOp)> getWarpSize = nullptr);
+/// Pass to specialize workgroup distribution loops
+std::unique_ptr<OperationPass<func::FuncOp>>
+createWorkgroupSpecializationPass();
+
/// Converts vector ops to gpu dialect.
std::unique_ptr<OperationPass<func::FuncOp>>
createWorkGroupSwizzle(unsigned swizzleLogTile = 0);
+/// Register Common GPU passes.
+void registerCodegenCommonGPUPasses();
+
} // namespace iree_compiler
} // namespace mlir
-#endif // IREE_COMPILER_CODEGEN_COMMON_GPU_COMMONGPUASSES_H_
+#endif // IREE_COMPILER_CODEGEN_COMMON_GPU_PASSES_H_
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/Passes.td b/compiler/src/iree/compiler/Codegen/Common/GPU/Passes.td
new file mode 100644
index 0000000..02c351a
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/Passes.td
@@ -0,0 +1,123 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef IREE_CODEGEN_COMMON_GPU_PASSES
+#define IREE_CODEGEN_COMMON_GPU_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+//===---------------------------------------------------------------------===//
+// Common Passes used for GPU-like backends (keep alphabetical)
+//===---------------------------------------------------------------------===//
+
+def GPUCheckResourceUsage :
+ Pass<"iree-codegen-gpu-check-resource-usage", "ModuleOp"> {
+ let summary = "Checks GPU specific resource usage constraints like shared memory limits";
+ let constructor = "mlir::iree_compiler::createGPUCheckResourceUsagePass()";
+}
+
+def GPUDistribute :
+ Pass<"iree-codegen-gpu-distribute", "func::FuncOp"> {
+ let summary = "Pass to distribute scf.forall ops.";
+ let constructor = "mlir::iree_compiler::createGPUDistribute()";
+}
+
+def GPUDistributeSharedMemoryCopy :
+ Pass<"iree-codegen-gpu-distribute-shared-memory-copy", "func::FuncOp"> {
+ let summary = "Pass to distribute shared memory copies to threads.";
+ let constructor = "mlir::iree_compiler::createGPUDistributeSharedMemoryCopy()";
+}
+
+def GPUMultiBuffering :
+ Pass<"iree-codegen-gpu-multi-buffering", "func::FuncOp"> {
+ let summary = "Pass to do multi buffering.";
+ let constructor = "mlir::iree_compiler::createGPUMultiBuffering()";
+}
+
+def GPUPipelining : Pass<"iree-codegen-gpu-pipelining", "func::FuncOp"> {
+ let summary = "Pass to do software pipelining.";
+ let constructor = "mlir::iree_compiler::createGPUPipeliningPass()";
+ let options = [
+ Option<"epiloguePeeling", "epilogue-peeling", "bool",
+ /*default=*/"true",
+ "Try to use un-peeling epilogue when false, peeled epilouge o.w.">,
+ Option<"depth", "pipeline-depth", "int64_t",
+ /*default=*/"2",
+ "Number of stages ">,
+ Option<"scheduleIndex", "schedule-index", "int64_t",
+ /*default=*/"0",
+ "Allows picking different schedule for the pipelining transformation.">,
+ Option<"transformFileName", "transform-file-name", "std::string",
+ /*default=*/"\"\"",
+ "Optional filename containing a transform dialect specification to "
+ "apply. If left empty, the IR is assumed to contain one top-level "
+ "transform dialect operation somewhere in the module.">,
+ ];
+}
+
+def GPUReduceBankConflicts :
+ Pass<"iree-codegen-gpu-reduce-bank-conflicts", "func::FuncOp"> {
+ let summary = "Pass to try to reduce the number of bank conflicts.";
+ let constructor = "mlir::iree_compiler::createGPUReduceSharedMemoryBankConflicts()";
+}
+
+def GPUTensorAlloc :
+ Pass<"iree-codegen-gpu-tensor-alloc", "func::FuncOp"> {
+ let summary = "Pass to tile reduction dimensions and create allocations for "
+ "some tensor values to use GPU shared memory";
+ let constructor = "mlir::iree_compiler::createGPUTensorAlloc()";
+}
+
+def GPUTensorTile :
+ Pass<"iree-codegen-gpu-tensor-tile", "func::FuncOp"> {
+ let summary = "Pass to tile tensor (linalg) ops within a GPU workgroup";
+ let constructor = "mlir::iree_compiler::createGPUTensorTile()";
+}
+
+def GPUTileReduction :
+ Pass<"iree-codegen-gpu-tile-reduction", "func::FuncOp"> {
+ let summary = "Pass to tile linalg reduction dimensions.";
+ let constructor = "mlir::iree_compiler::createGPUTileReductionPass()";
+}
+
+def GPUVectorization :
+ Pass<"iree-codegen-gpu-vectorization", "func::FuncOp"> {
+ let summary = "Pass to convert linalg into Vector.";
+ let constructor = "mlir::iree_compiler::createGPUVectorizationPass()";
+ let options = [
+ Option<"generateContract", "generate-contract", "bool",
+ /*default=*/"true",
+ "Try to convert reduction to vector.contract.">,
+ Option<"maxVectorSize", "max-vector-size", "int64_t",
+ /*default=*/"4096",
+ "Max vector size allowed to avoid creating large vectors.">
+ ];
+}
+
+def VectorReduceToGPU :
+ Pass<"iree-codegen-reduction-to-gpu", "func::FuncOp"> {
+ let summary = "Convert vector reduction to gpu ops.";
+ let constructor = "mlir::iree_compiler::createConvertVectorReductionToGPUPass()";
+}
+
+def WorkgroupSpecialization :
+ Pass<"iree-codegen-workgroup-specialization", "func::FuncOp"> {
+ let summary = "Specialize workgroup distribution loops";
+ let constructor = "mlir::iree_compiler::createWorkgroupSpecializationPass()";
+}
+
+def WorkGroupSwizzle :
+ Pass<"iree-workgroup-swizzle", "func::FuncOp"> {
+ let summary = "swizzle the workgroup ids for better cache reuse";
+ let constructor = "mlir::iree_compiler::createWorkGroupSwizzle()";
+ let options = [
+ Option<"logTile", "logTile", "unsigned",
+ /*default=*/"0",
+ "pass the tile value for unit testing">,
+ ];
+}
+
+#endif // IREE_CODEGEN_COMMON_GPU_PASSES
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/VectorReductionToGPU.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/VectorReductionToGPU.cpp
index 96581d6..9f8c7e5 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/VectorReductionToGPU.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/VectorReductionToGPU.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/GPU/WorkGroupSwizzle.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/WorkGroupSwizzle.cpp
index 1cf724d..55b70a5 100644
--- a/compiler/src/iree/compiler/Codegen/Common/GPU/WorkGroupSwizzle.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/WorkGroupSwizzle.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/WorkgroupSpecializationPass.cpp b/compiler/src/iree/compiler/Codegen/Common/GPU/WorkgroupSpecializationPass.cpp
similarity index 96%
rename from compiler/src/iree/compiler/Codegen/Common/WorkgroupSpecializationPass.cpp
rename to compiler/src/iree/compiler/Codegen/Common/GPU/WorkgroupSpecializationPass.cpp
index 86b03e9..53cde87 100644
--- a/compiler/src/iree/compiler/Codegen/Common/WorkgroupSpecializationPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/GPU/WorkgroupSpecializationPass.cpp
@@ -28,10 +28,8 @@
//
//===---------------------------------------------------------------------===//
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtOps.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/Common/Transforms.h"
-#include "iree/compiler/Codegen/Interfaces/PartitionableLoopsInterface.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Debug.h"
@@ -39,6 +37,7 @@
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
+#include "mlir/Dialect/Tensor/IR/Tensor.h"
#define DEBUG_TYPE "iree-codegen-workgroup-specialization"
diff --git a/compiler/src/iree/compiler/Codegen/Common/HoistStaticallyBoundAllocations.cpp b/compiler/src/iree/compiler/Codegen/Common/HoistStaticallyBoundAllocations.cpp
index 33f084a..ab2dafe 100644
--- a/compiler/src/iree/compiler/Codegen/Common/HoistStaticallyBoundAllocations.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/HoistStaticallyBoundAllocations.cpp
@@ -4,9 +4,9 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Common/Transforms.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/IREEComprehensiveBufferizePass.cpp b/compiler/src/iree/compiler/Codegen/Common/IREEComprehensiveBufferizePass.cpp
index 185624d..dc26664 100644
--- a/compiler/src/iree/compiler/Codegen/Common/IREEComprehensiveBufferizePass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/IREEComprehensiveBufferizePass.cpp
@@ -11,9 +11,9 @@
//===----------------------------------------------------------------------===//
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Interfaces/BufferizationInterfaces.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Dialect/Flow/IR/FlowDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "iree/compiler/Dialect/Util/IR/UtilDialect.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/IREEExpandStridedMetadata.cpp b/compiler/src/iree/compiler/Codegen/Common/IREEExpandStridedMetadata.cpp
index 39d46ca..b7e7eef 100644
--- a/compiler/src/iree/compiler/Codegen/Common/IREEExpandStridedMetadata.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/IREEExpandStridedMetadata.cpp
@@ -6,10 +6,10 @@
#define DEBUG_TYPE "iree-codegen-expand-strided-metadata"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
#include "iree/compiler/Codegen/Dialect/UKernelOps.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/MemRef/Transforms/Transforms.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/InstrumentMemoryAccesses.cpp b/compiler/src/iree/compiler/Codegen/Common/InstrumentMemoryAccesses.cpp
index 136df74..d73adce 100644
--- a/compiler/src/iree/compiler/Codegen/Common/InstrumentMemoryAccesses.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/InstrumentMemoryAccesses.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "llvm/ADT/TypeSwitch.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/LowerUKernelsToCalls.cpp b/compiler/src/iree/compiler/Codegen/Common/LowerUKernelsToCalls.cpp
index 255ea31..ae361c1 100644
--- a/compiler/src/iree/compiler/Codegen/Common/LowerUKernelsToCalls.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/LowerUKernelsToCalls.cpp
@@ -4,9 +4,9 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Interfaces/UKernelOpInterface.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "llvm/ADT/MapVector.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/MaterializeEncodingIntoPackUnPack.cpp b/compiler/src/iree/compiler/Codegen/Common/MaterializeEncodingIntoPackUnPack.cpp
index 1af52c0..969ec1a 100644
--- a/compiler/src/iree/compiler/Codegen/Common/MaterializeEncodingIntoPackUnPack.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/MaterializeEncodingIntoPackUnPack.cpp
@@ -10,10 +10,10 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
#include "iree/compiler/Codegen/Common/EncodingInfo.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenOps.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
#include "iree/compiler/Dialect/HAL/IR/HALTypes.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/MemrefCopyToLinalg.cpp b/compiler/src/iree/compiler/Codegen/Common/MemrefCopyToLinalg.cpp
index cd78171..c6856c0 100644
--- a/compiler/src/iree/compiler/Codegen/Common/MemrefCopyToLinalg.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/MemrefCopyToLinalg.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/OptimizeVectorTransferPass.cpp b/compiler/src/iree/compiler/Codegen/Common/OptimizeVectorTransferPass.cpp
index 806455c..cb96188 100644
--- a/compiler/src/iree/compiler/Codegen/Common/OptimizeVectorTransferPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/OptimizeVectorTransferPass.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/PadDynamicAlloc.cpp b/compiler/src/iree/compiler/Codegen/Common/PadDynamicAlloc.cpp
index e390ba6..64fea16 100644
--- a/compiler/src/iree/compiler/Codegen/Common/PadDynamicAlloc.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/PadDynamicAlloc.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/compiler/src/iree/compiler/Codegen/PassDetail.h b/compiler/src/iree/compiler/Codegen/Common/PassDetail.h
similarity index 69%
copy from compiler/src/iree/compiler/Codegen/PassDetail.h
copy to compiler/src/iree/compiler/Codegen/Common/PassDetail.h
index fea75f0..2f74e83 100644
--- a/compiler/src/iree/compiler/Codegen/PassDetail.h
+++ b/compiler/src/iree/compiler/Codegen/Common/PassDetail.h
@@ -1,11 +1,11 @@
-// Copyright 2021 The IREE Authors
+// Copyright 2023 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#ifndef IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
-#define IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#ifndef IREE_COMPILER_CODEGEN_COMMON_PASS_DETAIL_H_
+#define IREE_COMPILER_CODEGEN_COMMON_PASS_DETAIL_H_
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
@@ -18,9 +18,9 @@
namespace iree_compiler {
#define GEN_PASS_CLASSES
-#include "iree/compiler/Codegen/Passes.h.inc"
+#include "iree/compiler/Codegen/Common/Passes.h.inc"
} // namespace iree_compiler
} // namespace mlir
-#endif // IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#endif // IREE_COMPILER_CODEGEN_COMMON_PASS_DETAIL_H_
diff --git a/compiler/src/iree/compiler/Codegen/Common/Passes.cpp b/compiler/src/iree/compiler/Codegen/Common/Passes.cpp
new file mode 100644
index 0000000..14dee6a
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/Common/Passes.cpp
@@ -0,0 +1,38 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
+#include "mlir/Pass/PassManager.h"
+
+namespace mlir {
+namespace iree_compiler {
+
+void addCommonTargetExecutablePreprocessingPasses(OpPassManager &passManager) {
+ passManager.addNestedPass<func::FuncOp>(createTypePropagationPass());
+ passManager.addPass(createBubbleUpOrdinalOpsPass());
+ passManager.addPass(createBufferizeCopyOnlyDispatchesPass());
+ passManager.addNestedPass<func::FuncOp>(
+ IREE::LinalgExt::createDecomposeSoftmaxPass());
+ // Temporary solution to avoid large allocations due to softmax lowering.
+ passManager.addNestedPass<func::FuncOp>(createRematerializeParallelOpsPass());
+}
+
+//===---------------------------------------------------------------------===//
+// Register Common Passes
+//===---------------------------------------------------------------------===//
+
+namespace {
+#define GEN_PASS_REGISTRATION
+#include "iree/compiler/Codegen/Common/Passes.h.inc"
+} // namespace
+
+void registerCodegenCommonPasses() {
+ // Generated.
+ registerPasses();
+}
+} // namespace iree_compiler
+} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/Common/CommonPasses.h b/compiler/src/iree/compiler/Codegen/Common/Passes.h
similarity index 97%
rename from compiler/src/iree/compiler/Codegen/Common/CommonPasses.h
rename to compiler/src/iree/compiler/Codegen/Common/Passes.h
index 76076b8..165b378 100644
--- a/compiler/src/iree/compiler/Codegen/Common/CommonPasses.h
+++ b/compiler/src/iree/compiler/Codegen/Common/Passes.h
@@ -18,6 +18,7 @@
namespace mlir {
namespace iree_compiler {
+
/// Passes that are done on all backends before target-specific code-generation
/// kicks in.
void addCommonTargetExecutablePreprocessingPasses(OpPassManager &passManager);
@@ -97,6 +98,10 @@
// Extract address computations into their own separate instructions.
std::unique_ptr<Pass> createExtractAddressComputationPass();
+// Extract address computations (including the ones with GPU instructions) into
+// their own separate instructions.
+std::unique_ptr<Pass> createExtractAddressComputationGPUPass();
+
/// Flattens n-D MemRef subspan ops to 1-D MemRef and folds the byte offsets
/// on subspan ops to the consumer load/store ops, in preparation for lowering
/// to backends that require linearized access.
@@ -205,10 +210,6 @@
/// control flows.
std::unique_ptr<OperationPass<func::FuncOp>> createVectorizePadPass();
-/// Pass to specialize workgroup distribution loops
-std::unique_ptr<OperationPass<func::FuncOp>>
-createWorkgroupSpecializationPass();
-
/// Erases #hal.descriptor_type as MemRef memory space.
LogicalResult eraseHALDescriptorTypeFromMemRef(func::FuncOp funcOp);
@@ -241,6 +242,9 @@
void populateVectorizePadPatterns(RewritePatternSet &patterns,
PatternBenefit baseBenefit = 1);
+/// Method to register all passes.
+void registerCodegenCommonPasses();
+
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/Common/Passes.td b/compiler/src/iree/compiler/Codegen/Common/Passes.td
new file mode 100644
index 0000000..3aa73da
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/Common/Passes.td
@@ -0,0 +1,401 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef IREE_CODEGEN_COMMON_PASSES
+#define IREE_CODEGEN_COMMON_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+//===---------------------------------------------------------------------===//
+// Common passes for all backends (keep alphabetical)
+//===---------------------------------------------------------------------===//
+
+def BubbleUpOrdinalOps : Pass<"iree-codegen-bubble-up-ordinal-ops", ""> {
+ let summary = "Bubbles op ordinal ops to allow for workgroup count computation";
+ let constructor = "mlir::iree_compiler::createBubbleUpOrdinalOpsPass()";
+}
+
+def BufferizeCopyOnlyDispatches :
+ Pass<"iree-codegen-bufferize-copy-only-dispatches", "ModuleOp"> {
+ let summary =
+ "Bufferize dispatches that copy to/from interfaces to convert to a linalg.copy op";
+ let constructor = "mlir::iree_compiler::createBufferizeCopyOnlyDispatchesPass()";
+}
+
+def CleanupBufferAllocView :
+ Pass<"iree-codegen-cleanup-buffer-alloc-view", "func::FuncOp"> {
+ let summary =
+ "Performs cleanups over HAL interface/buffer allocation/view operations";
+ let constructor = "mlir::iree_compiler::createCleanupBufferAllocViewPass()";
+}
+
+def ConcretizePadResultShape :
+ Pass<"iree-codegen-concretize-pad-result-shape", "func::FuncOp"> {
+ let summary =
+ "Concretizes tensor.pad op's result shape if its source op"
+ "implements OffsetSizeAndStrideOpInterface.";
+ let constructor = "mlir::iree_compiler::createConcretizePadResultShapePass()";
+}
+
+def ConvertBf16ArithToF32 : Pass<"iree-convert-bf16-arith-to-f32", "ModuleOp"> {
+ let summary = "Convert bf16 arithmetic operations to f32";
+ let constructor = "mlir::iree_compiler::createConvertBf16ArithToF32Pass()";
+}
+
+def ConvertBf16ToUInt16Buffers :
+ Pass<"iree-convert-bf16-to-uint16-buffers", "ModuleOp"> {
+ let summary = "Convert bf16 buffers to uint16 equivalents";
+ let constructor = "mlir::iree_compiler::createConvertBf16ToUInt16BuffersPass()";
+}
+
+def ConvertToDestinationPassingStyle :
+ Pass<"iree-codegen-convert-to-destination-passing-style", "func::FuncOp"> {
+ let summary =
+ "Transforms the code to make the dispatch use destination-passing style";
+ let constructor = "mlir::iree_compiler::createConvertToDestinationPassingStylePass()";
+ let options = [
+ Option<"useWARForCooperativeMatrixCodegen", "use-war-for-cooperative-matrix-codegen",
+ "bool", /*default=*/"false",
+ "WAR for failure in Cooperative matrix codegen pipelines. See #10648.">
+ ];
+}
+
+def DecomposeAffineOps: Pass<"decompose-affine-ops"> {
+ let summary = "Decompose `affine.apply` operations into sub `affine.apply`";
+ let description = [{
+ Decompose `affine.apply` operations into sub `affine.apply` where each
+ sub expression references values that are defined in the same loop scope.
+ The sub expression are then stitched back together following the loop
+ nest order.
+
+ The goal of this pass is to break down `affine.apply` expressions such
+ that the resulting sub expressions can be hoisted out in their respective
+ loop.
+
+ E.g., Let's say we have
+ ```mlir
+ %res = affine.apply
+ affine_map<()[s0, s1, s2] -> (s0 * 1024 + s1 * 32 + s2)>()
+ [%loopVariant, %inv1, %inv2]
+ ```
+ Where `%inv1` and `%inv2` are loop invariant and `%loopVariant` is not.
+ This will produce the following subexpressions:
+ ```mlir
+ // Loop invariant computations first.
+ %inv1x32 =
+ affine.apply affine_map<()[s0] -> (s0 * 32)>()[%inv1]
+ %inv1x32_plus_inv2 =
+ affine.apply affine_map<()[s0, s1] -> (s0 + s1)>()[%inv1x32, %inv2]
+ // Loop variant computation next.
+ %loopVariantx1024 =
+ affine.apply affine_map<()[s0] -> (s0 * 1024)>()[%loopVariant]
+ // Compose things back together.
+ %res =
+ affine.apply affine_map<()[s0, s1] -> (s0 + s1)>()
+ [%loopVariant, %inv1x32_plus_inv2]
+ ```
+ Now the sequence of instructions leading to and including
+ `%inv1x32_plus_inv2` can be hoisted out of the loop.
+
+ This pass requires `scf.for` structures to still be around otherwise
+ the break down will be meaningless.
+
+ Note: The decomposition performed by this pass will be undone by
+ canonicalization. Make sure to lower the resulting ops before that.
+ }];
+ let constructor = "mlir::iree_compiler::createDecomposeAffineOpsPass()";
+ let dependentDialects = [
+ "affine::AffineDialect"
+ ];
+}
+
+def DecomposeConvolutionToLowerDimOps :
+ Pass<"iree-codegen-decompose-convolution-to-lower-dim-ops", ""> {
+ let summary = "Decomposes linalg convolution ops to lower dim ops";
+ let constructor =
+ "mlir::iree_compiler::createDecomposeConvolutionToLowerDimOpsPass()";
+}
+
+def DecomposeLinalgGeneric :
+ Pass<"iree-codegen-decompose-linalg-generic", ""> {
+ let summary = "Decomposes linalg generic ops into individual ops";
+ let description = [{
+ It is sometimes advantageous to operate on generic ops which contain
+ at most one non-yield body operation. This is most often the case when
+ needing to materialize individual ops (which some backends require).
+ Note that this is often an extreme pessimization unless if part of a
+ lowering flow which was designed for it.
+
+ Operates on tensor based linalg ops.
+ }];
+ let constructor = "mlir::iree_compiler::createDecomposeLinalgGenericPass()";
+}
+
+def DecomposePackUnPackOps :
+ Pass<"iree-codegen-decompose-pack-unpack-ops", "func::FuncOp"> {
+ let summary = "Decompose pack/unpack ops into vectorizable ops";
+ let constructor = "mlir::iree_compiler::createDecomposePackUnPackOpsPass()";
+ let options = [
+ Option<"tileOuterToOne", "tile-outer-to-one", "bool", "false",
+ "Always apply tiling to make outer dimension be ones">
+ ];
+}
+
+def EliminateEmptyTensors :
+ Pass<"iree-eliminate-empty-tensors", "ModuleOp"> {
+ let summary = "Eliminate tensor.empty ops to avoid buffer allocations";
+ let constructor = "mlir::iree_compiler::createEliminateEmptyTensorsPass()";
+}
+
+def EraseHALDescriptorTypeFromMemRef :
+ Pass<"iree-codegen-erase-hal-descriptor-type-from-memref", "func::FuncOp"> {
+ let summary = "Erase #hal.descriptor_type from MemRef memory space";
+ let constructor =
+ "mlir::iree_compiler::createEraseHALDescriptorTypeFromMemRefPass()";
+}
+
+def ExtractAddressComputation: Pass<"extract-address-computation"> {
+ let summary = "Extract address computations from memory accesses";
+ let description = [{
+ Extract the address computation from the instructions with memory
+ accesses such that these memory accesses use only a base pointer.
+
+ For instance,
+ ```mlir
+ memref.load %base[%off0, ...]
+ ```
+
+ Will be rewritten in:
+ ```mlir
+ %new_base = memref.subview %base[%off0,...][1,...][1,...]
+ memref.load %new_base[%c0,...]
+ ```
+ }];
+ let constructor = "mlir::iree_compiler::createExtractAddressComputationPass()";
+ let dependentDialects = [
+ "memref::MemRefDialect"
+ ];
+}
+
+def FlattenMemRefSubspan :
+ Pass<"iree-codegen-flatten-memref-subspan", "ModuleOp"> {
+ let summary =
+ "Flatten n-D MemRef subspan ops to 1-D ones and fold byte offsets";
+ let constructor = "mlir::iree_compiler::createFlattenMemRefSubspanPass()";
+}
+
+def FoldAffineMinInDistributedLoops :
+ Pass<"iree-codegen-fold-affinemin-in-distributed-loops", "func::FuncOp"> {
+ let summary = "Fold `affine.min` ops in distributed loops";
+ let constructor = "mlir::iree_compiler::createFoldAffineMinInDistributedLoopsPass()";
+}
+
+def FoldTensorExtractOp :
+ Pass<"iree-codegen-fold-tensor-extract-op", ""> {
+ let summary = "Fold `tensor.extract` operations prior to lowering to LLVM";
+ let constructor = "mlir::iree_compiler::createFoldTensorExtractOpPass()";
+}
+
+def ForOpCanonicalization :
+ Pass<"iree-codegen-canonicalize-scf-for", "func::FuncOp"> {
+ let summary =
+ "Adhoc canonicalization of selected loop-carried values/dependencies for scf.for ops";
+ let constructor = "mlir::iree_compiler::createForOpCanonicalizationPass()";
+}
+
+def FuseTensorPadWithConsumer :
+ Pass<"iree-codegen-fuse-tensor-pad-with-consumer", "func::FuncOp"> {
+ let summary = "Fuse tensor.pad op into its consumer op's tiled loop nest";
+ let constructor = "mlir::iree_compiler::createFuseTensorPadWithConsumerPass()";
+}
+
+def HoistStaticallyBoundAllocations :
+ Pass<"iree-hoist-statically-bound-allocations", "func::FuncOp"> {
+ let summary = "Hoist statically bound alloca ops to the entry block of functions";
+ let constructor = "mlir::iree_compiler::createHoistStaticallyBoundAllocationsPass()";
+}
+
+def IREEComprehensiveBufferize :
+ Pass<"iree-codegen-iree-comprehensive-bufferize", "ModuleOp"> {
+ let summary = "Convert from to Linalg ops on tensors to buffers";
+ let constructor = "mlir::iree_compiler::createIREEComprehensiveBufferizePass()";
+ let options = [
+ Option<"testAnalysisOnly", "test-analysis-only", "bool",
+ /*default=*/"false",
+ "Only runs inplaceability analysis (for testing purposes only)">,
+ Option<"printConflicts", "print-conflicts", "bool",
+ /*default=*/"false",
+ "Annotates IR with RaW conflicts. Requires test-analysis-only.">,
+ ];
+}
+
+def IREEExpandStridedMetadata :
+ Pass<"iree-codegen-expand-strided-metadata", ""> {
+ let summary = "Resolve memref.extract_strided_metadata operations";
+ let constructor = "mlir::iree_compiler::createIREEExpandStridedMetadataPass()";
+ let options = [
+ Option<"allowUnresolved", "allow-unresolved", "bool", /*default=*/"false",
+ "Allow unresolved strided metadata op (for testing)">,
+ ];
+}
+
+def InstrumentMemoryAccesses :
+ Pass<"iree-codegen-instrument-memory-accesses", "func::FuncOp"> {
+ let summary = "Instruments memory reads and writes for address tracking when dispatch instrumentation is enabled.";
+ let constructor = "mlir::iree_compiler::createInstrumentMemoryAccessesPass()";
+}
+
+def LowerUKernelOpsToCalls :
+ Pass<"iree-codegen-lower-ukernel-ops-to-calls", "ModuleOp"> {
+ let summary = "Lower micro-kernel wrapper ops into function calls";
+ let constructor = "mlir::iree_compiler::createLowerUKernelOpsToCallsPass()";
+}
+
+def MemrefCopyToLinalgPass :
+ Pass<"iree-codegen-memrefcopy-to-linalg", "func::FuncOp"> {
+ let summary = "Convert memref.copy to linalg op";
+ let constructor =
+ "mlir::iree_compiler::createMemrefCopyToLinalgPass()";
+}
+
+def OptimizeVectorTransfer :
+ Pass<"iree-codegen-optimize-vector-transfer", "func::FuncOp"> {
+ let summary =
+ "Run optimization transformations on vector transfer operations";
+ let constructor = "mlir::iree_compiler::createOptimizeVectorTransferPass()";
+ let options = [
+ Option<"optionFlatten", "flatten", "bool", "false",
+ "Flatten the vector type of vector transfers where possible (contiguous row-major data).">,
+ Option<"optionDropUnitDims", "drop-unit-dims", "bool", /*default=*/"true",
+ "Drop unit dims in vector transfers where possible (might generate vector.shape_cast).">,
+ ];
+ let dependentDialects = [
+ "memref::MemRefDialect"
+ ];
+}
+
+def PadDynamicAlloc :
+ Pass<"iree-codegen-pad-dynamic-alloc", "func::FuncOp"> {
+ let summary = "Pass to pad dynamic alloc into static one.";
+ let constructor = "mlir::iree_compiler::createPadDynamicAlloc()";
+}
+
+def PolynomialApproximationPass :
+ Pass<"iree-codegen-polynomial-approximation", ""> {
+ let summary = "Convert math operations to their polynomial approximation";
+ let constructor =
+ "mlir::iree_compiler::createPolynomialApproximationPass()";
+}
+
+def RematerializeParallelOps :
+ Pass<"iree-codegen-rematerialize-parallel-ops", "func::FuncOp"> {
+ let summary = "Pass to rematerialize and merge parallel ops to avoid creating temporary allocs.";
+ let constructor = "mlir::iree_compiler::createRematerializeParallelOpsPass()";
+}
+
+def RemoveSingleIterationLoop :
+ Pass<"iree-codegen-remove-single-iteration-loop", "func::FuncOp"> {
+ let summary = "Remove distributed loop with single iteration.";
+ let constructor = "mlir::iree_compiler::createRemoveSingleIterationLoopPass()";
+}
+
+def SplitFullPartialTransfer :
+ Pass<"iree-codegen-split-full-partial-transfer", "func::FuncOp"> {
+ let summary =
+ "Split a vector.transfer operation into an in-bounds (i.e., no "
+ "out-of-bounds masking) fastpath and a slowpath.";
+ let constructor = "mlir::iree_compiler::createSplitFullPartialTransferPass()";
+ let options = [
+ Option<"splitVectorTransfersTo", "split-transfers", "std::string",
+ /*default=*/"",
+ [{Split vector transfers between slow (masked) and fast "
+ "(unmasked) variants. Possible options are:\n"
+ "\tnone [default]: keep unsplit vector.transfer and pay the price\n"
+ "\tlinalg-copy: use linalg.fill + linalg.generic for the slow path\n"
+ "\tvector-transfers: use extra small unmasked vector.transfers for"
+ " the slow path\n}]>,
+ ];
+}
+
+def TensorToVectorVectorizePad :
+ Pass<"iree-codegen-vectorize-tensor-pad", "func::FuncOp"> {
+ let summary = "Vectorize a very specific form of tensor.pad with "
+ "control flows";
+ let constructor =
+ "mlir::iree_compiler::createVectorizePadPass()";
+}
+
+def TestExecutablePreprocessing :
+ Pass<"iree-codegen-test-executable-preprocessing", ""> {
+ let summary = "Tests iree-hal-preprocess-executables-with behavior.";
+ let constructor = "mlir::iree_compiler::createTestExecutablePreprocessingPass()";
+}
+
+def TestPartitionableLoopsInterface :
+ Pass<"iree-codegen-test-partitionable-loops-interface", ""> {
+ let summary = "Test the PartitionableLoopsInterface";
+ let constructor = "mlir::iree_compiler::createTestPartitionableLoopsInterfacePass()";
+}
+
+def TileAndDistributeToWorkgroups :
+ Pass<"iree-codegen-tile-and-distribute-to-workgroups", "IREE::HAL::ExecutableVariantOp"> {
+ let summary = "Tile and distribute operations to workgroups";
+ let constructor = "mlir::iree_compiler::createTileAndDistributeToWorkgroupsPass()";
+ let options = [
+ Option<"maxWorkgroupParallelDims", "max-workgroup-parallel-dims", "int32_t",
+ /*default=*/ "",
+ "Maximum number of dims to distribute workgroups across.">,
+ Option<"distributionMethod", "distribution-method", "int32_t",
+ /*default=*/ "0",
+ "Pick the distribution method">
+ ];
+}
+
+def TransformDialectInterpreter :
+ Pass<"iree-transform-dialect-interpreter"> {
+ let summary = "Pass to apply transform dialect operations.";
+ let constructor =
+ "mlir::iree_compiler::createTransformDialectInterpreterPass()";
+ let options = [
+ Option<"transformFileName", "transform-file-name", "std::string",
+ /*default=*/"\"\"",
+ "Optional filename containing a transform dialect specification to "
+ "apply. If left empty, the IR is assumed to contain one top-level "
+ "transform dialect operation somewhere in the module.">,
+ Option<"transformLibraryFileName",
+ "transform-library-file-name",
+ "std::string",
+ /*default=*/"\"\"",
+ "If non-empty, the name of the file containing definitions of "
+ "external symbols referenced in the transform script. "
+ "These definitions will be used to replace declarations.">,
+ Option<"debugPayloadRootTag", "debug-payload-root-tag", "std::string",
+ /*default=*/"\"\"",
+ "Select the operation with 'transform.target_tag' attribute having "
+ "the given value as payload IR root. This allows user control on "
+ "what operation to transform in debug mode, without requiring "
+ "intimate knowledge of the IREE nested pass pipeline.\\n"
+ "If empty (normal operation mode), select the pass anchor "
+ "operation in the IREE pipeline, as the payload IR root.">,
+ Option<"debugTransformRootTag", "debug-transform-root-tag", "std::string",
+ /*default=*/"\"\"",
+ "Select the operation with 'transform.target_tag' attribute having "
+ "the given value as container IR for top-level transform ops. This "
+ "allows user control on what transformation to apply in debug "
+ "mode, without requiring intimate knowledge of the IREE nested "
+ "pass pipeline.\\n"
+ "If empty (normal operation mode), select the container of the "
+ "top-level transform op.">
+ ];
+}
+
+def TypePropagation :
+ Pass<"iree-codegen-type-propagation", "func::FuncOp"> {
+ let summary = "Propogate the type of tensor to avoid load/stores of illegal bit widths";
+ let constructor = "mlir::iree_compiler::createTypePropagationPass()";
+}
+
+#endif // IREE_CODEGEN_COMMON_PASSES
\ No newline at end of file
diff --git a/compiler/src/iree/compiler/Codegen/Common/PolynomialApproximationPass.cpp b/compiler/src/iree/compiler/Codegen/Common/PolynomialApproximationPass.cpp
index 960395d..bf5a082 100644
--- a/compiler/src/iree/compiler/Codegen/Common/PolynomialApproximationPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/PolynomialApproximationPass.cpp
@@ -3,8 +3,8 @@
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Math/Transforms/Approximation.h"
#include "mlir/Dialect/Math/Transforms/Passes.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/RematerializeParallelOps.cpp b/compiler/src/iree/compiler/Codegen/Common/RematerializeParallelOps.cpp
index 9ea756e..5ff8a40 100644
--- a/compiler/src/iree/compiler/Codegen/Common/RematerializeParallelOps.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/RematerializeParallelOps.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/RemoveTrivialLoops.cpp b/compiler/src/iree/compiler/Codegen/Common/RemoveTrivialLoops.cpp
index 8f4038f..3942891 100644
--- a/compiler/src/iree/compiler/Codegen/Common/RemoveTrivialLoops.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/RemoveTrivialLoops.cpp
@@ -4,9 +4,9 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "llvm/Support/Debug.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/SplitFullPartialTransferPass.cpp b/compiler/src/iree/compiler/Codegen/Common/SplitFullPartialTransferPass.cpp
index 34fa27d..7249c45 100644
--- a/compiler/src/iree/compiler/Codegen/Common/SplitFullPartialTransferPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/SplitFullPartialTransferPass.cpp
@@ -6,8 +6,8 @@
#include <string>
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/Transforms/VectorTransforms.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/TestExecutablePreprocessing.cpp b/compiler/src/iree/compiler/Codegen/Common/TestExecutablePreprocessing.cpp
index 9f4f8e8..751cbbf 100644
--- a/compiler/src/iree/compiler/Codegen/Common/TestExecutablePreprocessing.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/TestExecutablePreprocessing.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/TestPartitionableLoopsInterface.cpp b/compiler/src/iree/compiler/Codegen/Common/TestPartitionableLoopsInterface.cpp
index 41b78d4..6b28181 100644
--- a/compiler/src/iree/compiler/Codegen/Common/TestPartitionableLoopsInterface.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/TestPartitionableLoopsInterface.cpp
@@ -4,9 +4,9 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Interfaces/PartitionableLoopsInterface.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Dialect/Util/IR/UtilDialect.h"
#include "iree/compiler/Dialect/Util/IR/UtilOps.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/TileAndDistributeToWorkgroupsPass.cpp b/compiler/src/iree/compiler/Codegen/Common/TileAndDistributeToWorkgroupsPass.cpp
index 905a876..9e2c091 100644
--- a/compiler/src/iree/compiler/Codegen/Common/TileAndDistributeToWorkgroupsPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/TileAndDistributeToWorkgroupsPass.cpp
@@ -17,12 +17,12 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtOps.h"
#include "iree-dialects/Dialect/LinalgExt/Passes/Transforms.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
#include "iree/compiler/Codegen/Common/EncodingInfo.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Common/Transforms.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
#include "iree/compiler/Codegen/Interfaces/PartitionableLoopsInterface.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/TransformDialectInterpreterPass.cpp b/compiler/src/iree/compiler/Codegen/Common/TransformDialectInterpreterPass.cpp
index 18a102a..c162c3c 100644
--- a/compiler/src/iree/compiler/Codegen/Common/TransformDialectInterpreterPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/TransformDialectInterpreterPass.cpp
@@ -8,10 +8,10 @@
#include "iree-dialects/Dialect/LinalgExt/TransformOps/LinalgExtTransformOps.h"
#include "iree-dialects/Dialect/LinalgTransform/LinalgTransformOps.h"
#include "iree-dialects/Dialect/LinalgTransform/StructuredTransformOpsExt.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
#include "iree/compiler/Codegen/Common/TransformExtensions/CommonExtensions.h"
#include "iree/compiler/Codegen/LLVMCPU/TransformExtensions/LLVMCPUExtensions.h"
#include "iree/compiler/Codegen/LLVMGPU/TransformExtensions/LLVMGPUExtensions.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Dialect/Flow/IR/FlowDialect.h"
#include "iree/compiler/Dialect/Flow/TransformExtensions/FlowExtensions.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/BUILD.bazel b/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/BUILD.bazel
index 8a1fea0..ba932de 100644
--- a/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/BUILD.bazel
@@ -62,7 +62,7 @@
],
deps = [
":CommonExtensionsOpGen",
- "//compiler/src/iree/compiler/Codegen/Common:CommonPasses",
+ "//compiler/src/iree/compiler/Codegen/Common",
"//compiler/src/iree/compiler/Codegen/Interfaces:BufferizationInterfaces",
"//compiler/src/iree/compiler/Codegen/Transforms",
"//compiler/src/iree/compiler/Codegen/Utils",
diff --git a/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/CMakeLists.txt
index 0ec09f6..299b075 100644
--- a/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/CMakeLists.txt
@@ -63,7 +63,7 @@
MLIRVectorDialect
MLIRVectorToGPU
MLIRVectorTransforms
- iree::compiler::Codegen::Common::CommonPasses
+ iree::compiler::Codegen::Common
iree::compiler::Codegen::Interfaces::BufferizationInterfaces
iree::compiler::Codegen::Transforms
iree::compiler::Codegen::Utils
diff --git a/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/CommonExtensions.cpp b/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/CommonExtensions.cpp
index 7917ecf..d89b06d 100644
--- a/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/CommonExtensions.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/TransformExtensions/CommonExtensions.cpp
@@ -11,7 +11,7 @@
#include "iree-dialects/Dialect/LinalgTransform/StructuredTransformOpsExt.h"
#include "iree-dialects/Transforms/ListenerCSE.h"
#include "iree-dialects/Transforms/TransformMatchers.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Common/Transforms.h"
#include "iree/compiler/Codegen/Interfaces/BufferizationInterfaces.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/Transforms.h b/compiler/src/iree/compiler/Codegen/Common/Transforms.h
index 20d4f9d..dee5839 100644
--- a/compiler/src/iree/compiler/Codegen/Common/Transforms.h
+++ b/compiler/src/iree/compiler/Codegen/Common/Transforms.h
@@ -8,7 +8,7 @@
#define IREE_COMPILER_CODEGEN_COMMON_TRANSFORMS_H_
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Interfaces/BufferizationInterfaces.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/TypePropagationPass.cpp b/compiler/src/iree/compiler/Codegen/Common/TypePropagationPass.cpp
index 16d2c9a..10c7a5d 100644
--- a/compiler/src/iree/compiler/Codegen/Common/TypePropagationPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/TypePropagationPass.cpp
@@ -26,8 +26,8 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtOps.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Dialect/Util/IR/UtilTypes.h"
#include "iree/compiler/Utils/ElementPackingUtils.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/compiler/src/iree/compiler/Codegen/Common/VectorizePad.cpp b/compiler/src/iree/compiler/Codegen/Common/VectorizePad.cpp
index 355bca1..d3ba359 100644
--- a/compiler/src/iree/compiler/Codegen/Common/VectorizePad.cpp
+++ b/compiler/src/iree/compiler/Codegen/Common/VectorizePad.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel
index e27971f..8d4b9ec 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/BUILD.bazel
@@ -4,7 +4,7 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library")
+load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library", "iree_gentbl_cc_library")
package(
default_visibility = ["//visibility:public"],
@@ -12,6 +12,38 @@
licenses = ["notice"], # Apache 2.0
)
+iree_gentbl_cc_library(
+ name = "PassesIncGen",
+ tbl_outs = [
+ (
+ ["--gen-pass-decls"],
+ "Passes.h.inc",
+ ),
+ ],
+ tblgen = "@llvm-project//mlir:mlir-tblgen",
+ td_file = "Passes.td",
+ deps = ["@llvm-project//mlir:PassBaseTdFiles"],
+)
+
+iree_compiler_cc_library(
+ name = "PassHeaders",
+ hdrs = [
+ "PassDetail.h",
+ "Passes.h",
+ "Passes.h.inc",
+ ],
+ deps = [
+ ":PassesIncGen",
+ "//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
+ "//compiler/src/iree/compiler/Dialect/HAL/IR",
+ "//compiler/src/iree/compiler/Utils",
+ "@llvm-project//mlir:LinalgTransforms",
+ "@llvm-project//mlir:MemRefDialect",
+ "@llvm-project//mlir:Pass",
+ "@llvm-project//mlir:Transforms",
+ ],
+)
+
iree_compiler_cc_library(
name = "LLVMCPU",
srcs = [
@@ -47,15 +79,16 @@
hdrs = [
"DispatchABI.h",
"KernelDispatch.h",
- "LLVMCPUPasses.h",
+ "Passes.h",
"TargetMLTransformInfo.h",
"TileSizeSelection.h",
"Utils.h",
],
deps = [
- "//compiler/src/iree/compiler/Codegen:PassHeaders",
+ ":PassHeaders",
+ ":PassesIncGen",
"//compiler/src/iree/compiler/Codegen/Common",
- "//compiler/src/iree/compiler/Codegen/Common:CommonPasses",
+ "//compiler/src/iree/compiler/Codegen/Common:TransformDialectInterpreterPass",
"//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
"//compiler/src/iree/compiler/Codegen/Interfaces:PartitionableLoopsInterface",
"//compiler/src/iree/compiler/Codegen/Interfaces:UKernelOpInterface",
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt
index 25e42f8..fc68a04 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/CMakeLists.txt
@@ -10,13 +10,41 @@
iree_add_all_subdirs()
+iree_tablegen_library(
+ NAME
+ PassesIncGen
+ TD_FILE
+ "Passes.td"
+ OUTS
+ --gen-pass-decls Passes.h.inc
+)
+
+iree_cc_library(
+ NAME
+ PassHeaders
+ HDRS
+ "PassDetail.h"
+ "Passes.h"
+ "Passes.h.inc"
+ DEPS
+ ::PassesIncGen
+ MLIRLinalgTransforms
+ MLIRMemRefDialect
+ MLIRPass
+ MLIRTransforms
+ iree::compiler::Codegen::Dialect::IREECodegenDialect
+ iree::compiler::Dialect::HAL::IR
+ iree::compiler::Utils
+ PUBLIC
+)
+
iree_cc_library(
NAME
LLVMCPU
HDRS
"DispatchABI.h"
"KernelDispatch.h"
- "LLVMCPUPasses.h"
+ "Passes.h"
"TargetMLTransformInfo.h"
"TileSizeSelection.h"
"Utils.h"
@@ -50,6 +78,8 @@
"VectorContractCustomKernels.cpp"
"VerifyLinalgTransformLegality.cpp"
DEPS
+ ::PassHeaders
+ ::PassesIncGen
IREELinalgExtDialect
IREELinalgExtPasses
IREELinalgTransformDialect
@@ -108,11 +138,10 @@
MLIRVectorTransforms
iree::builtins::ukernel::exported_bits
iree::compiler::Codegen::Common
- iree::compiler::Codegen::Common::CommonPasses
+ iree::compiler::Codegen::Common::TransformDialectInterpreterPass
iree::compiler::Codegen::Dialect::IREECodegenDialect
iree::compiler::Codegen::Interfaces::PartitionableLoopsInterface
iree::compiler::Codegen::Interfaces::UKernelOpInterface
- iree::compiler::Codegen::PassHeaders
iree::compiler::Codegen::TransformStrategies::CPU
iree::compiler::Codegen::Transforms
iree::compiler::Codegen::Utils
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/ConvertToLLVM.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/ConvertToLLVM.cpp
index e87de0f..4f79451 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/ConvertToLLVM.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/ConvertToLLVM.cpp
@@ -5,9 +5,9 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "iree/compiler/Codegen/LLVMCPU/DispatchABI.h"
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/LLVMCPU/Utils.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/ExpandF16OpToF32Pass.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/ExpandF16OpToF32Pass.cpp
index c694da1..f2ce708 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/ExpandF16OpToF32Pass.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/ExpandF16OpToF32Pass.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignConstantOrdinals.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignConstantOrdinals.cpp
index 637b074..c365856 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignConstantOrdinals.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignConstantOrdinals.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignImportOrdinals.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignImportOrdinals.cpp
index 963f022..2980164 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignImportOrdinals.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUAssignImportOrdinals.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "llvm/ADT/MapVector.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUCheckIRBeforeLLVMConversion.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUCheckIRBeforeLLVMConversion.cpp
index ada489b..4badb9f 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUCheckIRBeforeLLVMConversion.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUCheckIRBeforeLLVMConversion.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "llvm/Support/CommandLine.h"
#include "mlir/Interfaces/ValueBoundsOpInterface.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUEmitVectorizationRemarks.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUEmitVectorizationRemarks.cpp
index de8b6d9..9d7b2ac 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUEmitVectorizationRemarks.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUEmitVectorizationRemarks.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULinkExecutables.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULinkExecutables.cpp
index 79afd7a..cd6f23e 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULinkExecutables.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULinkExecutables.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/Utils/LinkingUtils.h"
#include "iree/compiler/Utils/ModuleUtils.h"
#include "llvm/Support/FormatVariadic.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp
index 0bf6a85..445d84c 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerExecutableTarget.cpp
@@ -9,10 +9,10 @@
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
#include "iree/compiler/Codegen/LLVMCPU/KernelDispatch.h"
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/LLVMCPU/TileSizeSelection.h"
#include "iree/compiler/Codegen/LLVMCPU/Utils.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerToUKernels.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerToUKernels.cpp
index daf0993..01fd3e6 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerToUKernels.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPULowerToUKernels.cpp
@@ -8,8 +8,8 @@
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenOps.h"
#include "iree/compiler/Codegen/Dialect/UKernelOps.h"
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/Utils/EncodingUtils.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMaterializeEncodingPass.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMaterializeEncodingPass.cpp
index c2375d8..23abf5b 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMaterializeEncodingPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMaterializeEncodingPass.cpp
@@ -8,8 +8,8 @@
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
#include "iree/compiler/Codegen/Common/EncodingInfo.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
#include "iree/compiler/Codegen/LLVMCPU/Utils.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
#include "iree/compiler/Dialect/HAL/IR/HALTypes.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMmt4dVectorLowering.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMmt4dVectorLowering.cpp
index b3da29c..a058ec7 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMmt4dVectorLowering.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUMmt4dVectorLowering.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPeel.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPeel.cpp
index 7917f3a..d367cb6 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPeel.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPeel.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSplitReduction.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSplitReduction.cpp
index 05c2339..19694f2 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSplitReduction.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSplitReduction.cpp
@@ -4,9 +4,9 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/LLVMCPU/TileSizeSelection.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSynchronizeSymbolVisibility.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSynchronizeSymbolVisibility.cpp
index aeb94c4..41ca8f0 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSynchronizeSymbolVisibility.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUSynchronizeSymbolVisibility.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTensorPad.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTensorPad.cpp
index 566aed9..177d817 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTensorPad.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTensorPad.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/Linalg/Utils/Utils.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTile.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTile.cpp
index 2f1c06d..9d06fd1 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTile.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTile.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "llvm/Support/CommandLine.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTileAndFuse.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTileAndFuse.cpp
index 85e05d5..bf9d71a 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTileAndFuse.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUTileAndFuse.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "llvm/Support/CommandLine.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUUnfuseFMAOps.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUUnfuseFMAOps.cpp
index 916378a..efd745e 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUUnfuseFMAOps.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUUnfuseFMAOps.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/IR/Builders.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorLowering.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorLowering.cpp
index 5c711c9..42b4675 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorLowering.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorLowering.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorization.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorization.cpp
index 971889a..6abee23 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorization.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUVectorization.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Affine/LoopUtils.h"
#include "mlir/Dialect/Linalg/Transforms/Hoisting.h"
diff --git a/compiler/src/iree/compiler/Codegen/PassDetail.h b/compiler/src/iree/compiler/Codegen/LLVMCPU/PassDetail.h
similarity index 69%
copy from compiler/src/iree/compiler/Codegen/PassDetail.h
copy to compiler/src/iree/compiler/Codegen/LLVMCPU/PassDetail.h
index fea75f0..f057052 100644
--- a/compiler/src/iree/compiler/Codegen/PassDetail.h
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/PassDetail.h
@@ -1,11 +1,11 @@
-// Copyright 2021 The IREE Authors
+// Copyright 2023 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#ifndef IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
-#define IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#ifndef IREE_COMPILER_CODEGEN_LLVMCPU_PASS_DETAIL_H_
+#define IREE_COMPILER_CODEGEN_LLVMCPU_PASS_DETAIL_H_
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
@@ -18,9 +18,9 @@
namespace iree_compiler {
#define GEN_PASS_CLASSES
-#include "iree/compiler/Codegen/Passes.h.inc"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc"
} // namespace iree_compiler
} // namespace mlir
-#endif // IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#endif // IREE_COMPILER_CODEGEN_LLVMCPU_PASS_DETAIL_H_
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp
index 6831195..b2e0414 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.cpp
@@ -7,14 +7,14 @@
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
#include "iree-dialects/Dialect/LinalgTransform/Passes.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Interfaces/PartitionableLoopsInterface.h"
#include "iree/compiler/Codegen/LLVMCPU/KernelDispatch.h"
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/LLVMCPU/TileSizeSelection.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
-#include "iree/compiler/Codegen/VMVX/VMVXPasses.h"
+#include "iree/compiler/Codegen/VMVX/Passes.h"
#include "iree/compiler/Dialect/Util/Transforms/Passes.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/CommandLine.h"
@@ -768,5 +768,33 @@
variantPM.addPass(createLLVMCPUAssignImportOrdinalsPass());
}
+//===---------------------------------------------------------------------===//
+// Register LLVMCPU Passes
+//===---------------------------------------------------------------------===//
+
+namespace {
+#define GEN_PASS_REGISTRATION
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h.inc"
+} // namespace
+
+void registerCodegenLLVMCPUPasses() {
+ // Generated.
+ registerPasses();
+
+ static PassPipelineRegistration<> LinalgLLVMPipeline(
+ "iree-codegen-linalg-to-llvm-pipeline",
+ "Runs the progressive lowering pipeline from Linalg to LLVM",
+ [](OpPassManager &passManager) {
+ buildLLVMCPUCodegenPassPipeline(passManager);
+ });
+
+ static PassPipelineRegistration<> LLVMCPULinkingPipeline(
+ "iree-codegen-llvmcpu-linking-pipeline",
+ "Runs the LLVMCPU HAL executable linking pipeline",
+ [](OpPassManager &passManager) {
+ buildLLVMCPULinkingPassPipeline(passManager);
+ });
+}
+
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h
similarity index 97%
rename from compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h
rename to compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h
index 884054b..926f296 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.h
@@ -219,6 +219,12 @@
/// Populates passes needed to link HAL executables across LLVMCPU targets.
void buildLLVMCPULinkingPassPipeline(OpPassManager &passManager);
+//----------------------------------------------------------------------------//
+// Register LLVMCPU Passes
+//----------------------------------------------------------------------------//
+
+void registerCodegenLLVMCPUPasses();
+
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td
new file mode 100644
index 0000000..ca07c2a
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/Passes.td
@@ -0,0 +1,202 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef IREE_CODEGEN_LLVMCPU_PASSES
+#define IREE_CODEGEN_LLVMCPU_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+//------------------------------------------------------------------------------
+// LLVMCPU Passes (keep alphabetical)
+//------------------------------------------------------------------------------
+
+def ConvertToLLVM :
+ Pass<"iree-convert-to-llvm", "ModuleOp"> {
+ let summary =
+ "Perform final conversion from Linalg/HAL/Shape/Vector/Standard to LLVMIR dialect";
+ let constructor = "mlir::iree_compiler::createConvertToLLVMPass()";
+ let options = [
+ Option<"reassociateFpReductions", "reassociateFpReductions", "bool",
+ /*default=*/"false",
+ "Specifies if FP add and mult reductions can be reordered">,
+ ];
+}
+
+def ExpandArithF16ToF32 :
+ Pass<"iree-llvmcpu-expand-f16-op-to-f32", ""> {
+ let summary =
+ "Preform f16 opertaions by expanding them to f32.";
+ let constructor =
+ "mlir::iree_compiler::createExpandF16OpToF32Pass()";
+}
+
+def LLVMCPUAssignConstantOrdinals :
+ Pass<"iree-llvmcpu-assign-constant-ordinals", "IREE::HAL::ExecutableVariantOp"> {
+ let summary = "Assigns executable constant ordinals across all LLVMCPU variants.";
+ let constructor = "mlir::iree_compiler::createLLVMCPUAssignConstantOrdinalsPass()";
+}
+
+def LLVMCPUAssignImportOrdinals :
+ Pass<"iree-llvmcpu-assign-import-ordinals", "IREE::HAL::ExecutableVariantOp"> {
+ let summary = "Assigns executable import ordinals across all LLVMCPU variants.";
+ let constructor = "mlir::iree_compiler::createLLVMCPUAssignImportOrdinalsPass()";
+}
+
+def LLVMCPUCheckIRBeforeLLVMConversion :
+ Pass<"iree-llvmcpu-check-ir-before-llvm-conversion", "ModuleOp"> {
+ let summary = "Checks CPU backend specific IR constraints (like no allocas)";
+ let constructor = "mlir::iree_compiler::createLLVMCPUCheckIRBeforeLLVMConversionPass()";
+}
+
+def LLVMCPUEmitVectorizationRemarks :
+ Pass<"iree-llvmcpu-emit-vectorization-remarks", "func::FuncOp"> {
+ let summary = "Emit vectorization remarks on Linalg ops";
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPUEmitVectorizationRemarksPass()";
+}
+
+def LLVMCPULinkExecutables :
+ Pass<"iree-llvmcpu-link-executables", "mlir::ModuleOp"> {
+ let summary = "Links LLVMCPU HAL executables within the top-level program module.";
+ let constructor = "mlir::iree_compiler::createLLVMCPULinkExecutablesPass()";
+}
+
+def LLVMCPULowerExecutableTarget :
+ Pass<"iree-llvmcpu-lower-executable-target",
+ "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
+ let summary =
+ "Lower executable target using an IREE::HAL::DispatchLoweringPassPipeline";
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPULowerExecutableTargetPass()";
+}
+
+def LLVMCPULowerToUKernels :
+ Pass<"iree-llvmcpu-lower-to-ukernels", ""> {
+ let summary =
+ "Separate out parts of the IR that lower to a micro-kernel";
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPULowerToUKernelsPass()";
+}
+
+def LLVMCPUMaterializeEncoding :
+ Pass<"iree-llvmcpu-materialize-encoding", "func::FuncOp"> {
+ let summary = "Materialize the encoding for tensor as specified by the backend";
+ let constructor = "mlir::iree_compiler::createLLVMCPUMaterializeEncodingPass()";
+}
+
+def LLVMCPUMmt4dVectorLowering
+ : Pass<"iree-llvmcpu-mmt4d-vector-lowering", "func::FuncOp"> {
+ let summary = "Apply vector lowering logic to vector ops";
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPUMmt4dVectorLoweringPass()";
+}
+
+def LLVMCPUPeel :
+ Pass<"iree-llvmcpu-peel", "func::FuncOp"> {
+ let summary = "Pass to perform peeling on non-distributed loops.";
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPUPeelPass()";
+}
+
+def LLVMCPUSplitReduction : Pass<"iree-llvmcpu-split-reduction", "func::FuncOp"> {
+ let summary = "Pass to splitReduce linalg operations.";
+ let constructor = "mlir::iree_compiler::createLLVMCPUSplitReductionPass()";
+ let options = [
+ Option<"enableFpReductionReordering", "enable-fp-reduction-reordering",
+ "bool", /*default=*/"false",
+ "Flag to enable reduction reordering on floating points.">,
+ ];
+}
+
+def LLVMCPUSynchronizeSymbolVisibility :
+ Pass<"iree-llvmcpu-synchronize-symbol-visibility", "ModuleOp"> {
+ let summary = "Synchronizes LLVM linkage with MLIR symbol visibility";
+ let constructor = "mlir::iree_compiler::createLLVMCPUSynchronizeSymbolVisibilityPass()";
+}
+
+def LLVMCPUTensorPad :
+ Pass<"iree-llvmcpu-tensor-pad", "func::FuncOp"> {
+ let summary = "Pass to pad operations on tensors in top-down order.";
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPUTensorPadPass()";
+}
+
+def LLVMCPUTile :
+ Pass<"iree-llvmcpu-tile", "func::FuncOp"> {
+ let summary = "Pass to tile TilingInterface operations.";
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPUTilePass()";
+ let options = [
+ Option<"tilingLevel", "tiling-level", "int64_t", /*default=*/"-1",
+ "Use default tiling level used to retrieve the configuration from lowering_config">
+ ];
+}
+
+def LLVMCPUTileAndFuse :
+ Pass<"iree-llvmcpu-tile-and-fuse", "func::FuncOp"> {
+ let summary = "Pass to tile and fuse TilingInterface operations.";
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPUTileAndFusePass()";
+ let options = [
+ Option<"tilingLevel", "tiling-level", "int64_t", /*default=*/"-1",
+ "Use default tiling level used to retrieve the configuration from lowering_config">
+ ];
+}
+
+def LLVMCPUUnfuseFMAOps :
+ Pass<"iree-llvmcpu-unfuse-fma-pass", "func::FuncOp"> {
+ let summary = "Convert llvm.fma into unfused mulf and addf ops";
+ let constructor = "mlir::iree_compiler::createLLVMCPUUnfuseFMAOpsPass()";
+}
+
+def LLVMCPUVectorization :
+ Pass<"iree-llvmcpu-vectorization", "func::FuncOp"> {
+ let summary = "Pass to perform vectorization on tensor/linalg ops.";
+ let options = [
+ Option<"enableVectorMasking", "enable-vector-masking", "bool",/*default=*/"false",
+ "Enable vector masking during vectorization.">,
+ Option<"vectorizePadding", "vectorize-padding", "bool", /*default=*/"false",
+ "Rewrite all tensor.pad ops in the function to vector form.">,
+ Option<"vectorizeGatherAccesses", "vectorize-gather-accesses", "bool", /*default=*/"false",
+ "Enable vectorizaiton of operations that may generate vector.gather operations.">,
+ ];
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPUVectorizationPass()";
+}
+
+def LLVMCPUVectorLowering :
+ Pass<"iree-llvmcpu-vector-lowering", "func::FuncOp"> {
+ let summary = "Pass to lower Vector ops before conversion to LLVM.";
+ let options = [
+ Option<"splitVectorTransfersTo", "split-transfers", "std::string",
+ /*default=*/"",
+ [{Split vector transfers between slow (masked) and fast "
+ "(unmasked) variants. Possible options are:\n"
+ "\tnone [default]: keep unsplit vector.transfer and pay the price\n"
+ "\tlinalg-copy: use linalg.fill + linalg.generic for the slow path\n"
+ "\tvector-transfers: use extra small unmasked vector.transfers for"
+ " the slow path\n}]>,
+ Option<"lowerVectorTransposeToAVX2", "lower-vector-transpose-to-avx2", "bool",
+ /*default=*/"false",
+ "Add specific transpose to avx2 lowering patterns.">,
+ ];
+ let constructor =
+ "mlir::iree_compiler::createLLVMCPUVectorLoweringPass()";
+}
+
+def VectorContractCustomKernels :
+ Pass<"iree-llvmcpu-vector-contract-custom-kernels", "func::FuncOp"> {
+ let summary = "Enable custom kernels (inline assembly or intrinsics) for some vector.contract ops";
+ let constructor = "mlir::iree_compiler::createVectorContractCustomKernelsPass()";
+}
+
+def VerifyLinalgTransformLegality :
+ Pass<"iree-llvmcpu-verify-linalg-transform-legality", "ModuleOp"> {
+ let summary = "Verify that only supported IR constructs are passed to the compiler.";
+ let constructor = "mlir::iree_compiler::createVerifyLinalgTransformLegalityPass()";
+}
+
+#endif // IREE_CODEGEN_LLVMCPU_PASSES
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/VectorContractCustomKernels.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/VectorContractCustomKernels.cpp
index e9217f4..a4730ee 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/VectorContractCustomKernels.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/VectorContractCustomKernels.cpp
@@ -4,9 +4,9 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/LLVMCPU/Utils.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Utils/StringUtils.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/StringExtras.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMCPU/VerifyLinalgTransformLegality.cpp b/compiler/src/iree/compiler/Codegen/LLVMCPU/VerifyLinalgTransformLegality.cpp
index 660d4e3..39a13fa 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMCPU/VerifyLinalgTransformLegality.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMCPU/VerifyLinalgTransformLegality.cpp
@@ -5,8 +5,8 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/Dialect/Linalg/Transforms/Transforms.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/BUILD.bazel b/compiler/src/iree/compiler/Codegen/LLVMGPU/BUILD.bazel
index 3a4e1c0..799e688 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/BUILD.bazel
@@ -4,7 +4,7 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library")
+load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library", "iree_gentbl_cc_library")
package(
default_visibility = ["//visibility:public"],
@@ -12,6 +12,39 @@
licenses = ["notice"], # Apache 2.0
)
+iree_gentbl_cc_library(
+ name = "PassesIncGen",
+ tbl_outs = [
+ (
+ ["--gen-pass-decls"],
+ "Passes.h.inc",
+ ),
+ ],
+ tblgen = "@llvm-project//mlir:mlir-tblgen",
+ td_file = "Passes.td",
+ deps = ["@llvm-project//mlir:PassBaseTdFiles"],
+)
+
+iree_compiler_cc_library(
+ name = "PassHeaders",
+ hdrs = [
+ "PassDetail.h",
+ "Passes.h",
+ "Passes.h.inc",
+ ],
+ deps = [
+ ":PassesIncGen",
+ "//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
+ "//compiler/src/iree/compiler/Dialect/HAL/IR",
+ "//compiler/src/iree/compiler/Utils",
+ "@llvm-project//mlir:LinalgTransforms",
+ "@llvm-project//mlir:MemRefDialect",
+ "@llvm-project//mlir:NVGPUDialect",
+ "@llvm-project//mlir:Pass",
+ "@llvm-project//mlir:Transforms",
+ ],
+)
+
iree_compiler_cc_library(
name = "LLVMGPU",
srcs = [
@@ -34,12 +67,13 @@
hdrs = [
"ConvertToLLVM.h",
"KernelConfig.h",
- "LLVMGPUPasses.h",
+ "Passes.h",
],
deps = [
- "//compiler/src/iree/compiler/Codegen:PassHeaders",
+ ":PassHeaders",
+ ":PassesIncGen",
"//compiler/src/iree/compiler/Codegen/Common",
- "//compiler/src/iree/compiler/Codegen/Common:CommonPasses",
+ "//compiler/src/iree/compiler/Codegen/Common:TransformDialectInterpreterPass",
"//compiler/src/iree/compiler/Codegen/Common/GPU:CommonGPUPasses",
"//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
"//compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions:LLVMGPUExtensions",
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/LLVMGPU/CMakeLists.txt
index 4ef6d98..4800e70 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/CMakeLists.txt
@@ -10,13 +10,42 @@
iree_add_all_subdirs()
+iree_tablegen_library(
+ NAME
+ PassesIncGen
+ TD_FILE
+ "Passes.td"
+ OUTS
+ --gen-pass-decls Passes.h.inc
+)
+
+iree_cc_library(
+ NAME
+ PassHeaders
+ HDRS
+ "PassDetail.h"
+ "Passes.h"
+ "Passes.h.inc"
+ DEPS
+ ::PassesIncGen
+ MLIRLinalgTransforms
+ MLIRMemRefDialect
+ MLIRNVGPUDialect
+ MLIRPass
+ MLIRTransforms
+ iree::compiler::Codegen::Dialect::IREECodegenDialect
+ iree::compiler::Dialect::HAL::IR
+ iree::compiler::Utils
+ PUBLIC
+)
+
iree_cc_library(
NAME
LLVMGPU
HDRS
"ConvertToLLVM.h"
"KernelConfig.h"
- "LLVMGPUPasses.h"
+ "Passes.h"
SRCS
"ConvertToLLVM.cpp"
"ConvertToNVVM.cpp"
@@ -34,6 +63,8 @@
"Passes.cpp"
"Verifiers.cpp"
DEPS
+ ::PassHeaders
+ ::PassesIncGen
IREELinalgExtDialect
IREELinalgExtPasses
IREELinalgExtTransforms
@@ -91,12 +122,11 @@
MLIRVectorToSCF
MLIRVectorTransforms
iree::compiler::Codegen::Common
- iree::compiler::Codegen::Common::CommonPasses
iree::compiler::Codegen::Common::GPU::CommonGPUPasses
+ iree::compiler::Codegen::Common::TransformDialectInterpreterPass
iree::compiler::Codegen::Dialect::IREECodegenDialect
iree::compiler::Codegen::LLVMGPU::TransformExtensions::LLVMGPUExtensions
iree::compiler::Codegen::LLVMGPU::Utils
- iree::compiler::Codegen::PassHeaders
iree::compiler::Codegen::TransformStrategies::GPU
iree::compiler::Codegen::Transforms
iree::compiler::Codegen::Utils
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToLLVM.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToLLVM.cpp
index 9691407..17fea9d 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToLLVM.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToLLVM.cpp
@@ -6,8 +6,8 @@
#include "iree/compiler/Codegen/LLVMGPU/ConvertToLLVM.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/Util/IR/UtilOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToNVVM.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToNVVM.cpp
index 552da71..ed9cd75 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToNVVM.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToNVVM.cpp
@@ -5,8 +5,8 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "iree/compiler/Codegen/LLVMGPU/ConvertToLLVM.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/Util/IR/UtilOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToROCDL.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToROCDL.cpp
index a02c312..6175236 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToROCDL.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/ConvertToROCDL.cpp
@@ -5,8 +5,8 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "iree/compiler/Codegen/LLVMGPU/ConvertToLLVM.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/Util/IR/UtilOps.h"
#include "mlir/Conversion/ArithToLLVM/ArithToLLVM.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/ExtractAddressComputationGPUPass.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/ExtractAddressComputationGPUPass.cpp
index d34a8fa..1ef3051 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/ExtractAddressComputationGPUPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/ExtractAddressComputationGPUPass.cpp
@@ -5,8 +5,8 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "iree/compiler/Codegen/Common/ExtractAddressComputation.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUCastAddressSpaceFunction.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUCastAddressSpaceFunction.cpp
index 7ce7750..7f68c80 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUCastAddressSpaceFunction.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUCastAddressSpaceFunction.cpp
@@ -5,9 +5,8 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "iree/compiler/Codegen/LLVMGPU/KernelConfig.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
#include "iree/compiler/Codegen/LLVMGPU/TransformExtensions/LLVMGPUExtensions.h"
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "llvm/Support/raw_ostream.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPULowerExecutableTarget.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPULowerExecutableTarget.cpp
index ec3e3f1..64a5dbb 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPULowerExecutableTarget.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPULowerExecutableTarget.cpp
@@ -9,8 +9,8 @@
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
#include "iree/compiler/Codegen/LLVMGPU/KernelConfig.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUPackSharedMemoryAlloc.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUPackSharedMemoryAlloc.cpp
index 49d603e..7b2e880 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUPackSharedMemoryAlloc.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUPackSharedMemoryAlloc.cpp
@@ -6,9 +6,9 @@
#include <algorithm>
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/LLVMGPU/Utils/LLVMGPUUtils.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTensorCoreVectorization.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTensorCoreVectorization.cpp
index fdbdc28..dc89e08 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTensorCoreVectorization.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTensorCoreVectorization.cpp
@@ -8,8 +8,8 @@
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Common/GPU/GPUPatterns.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTensorPad.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTensorPad.cpp
index a374ed3..b22dd83 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTensorPad.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTensorPad.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/LinalgOpInfo.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTileAndDistribute.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTileAndDistribute.cpp
index 03bda5e..d12c309 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTileAndDistribute.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUTileAndDistribute.cpp
@@ -7,12 +7,12 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtOps.h"
#include "iree-dialects/Dialect/LinalgExt/Passes/Transforms.h"
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
#include "iree/compiler/Codegen/Common/GPU/GPUPatterns.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
#include "iree/compiler/Codegen/LLVMGPU/KernelConfig.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUVectorLowering.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUVectorLowering.cpp
index 473b284..7c1eb99 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUVectorLowering.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUVectorLowering.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "mlir/Conversion/VectorToSCF/VectorToSCF.h"
#include "mlir/Dialect/MemRef/Transforms/Transforms.h"
#include "mlir/Dialect/Vector/Transforms/LoweringPatterns.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUVectorToGPU.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUVectorToGPU.cpp
index 49a199e..ad533bb 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUVectorToGPU.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUVectorToGPU.cpp
@@ -5,9 +5,9 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "iree/compiler/Codegen/Common/GPU/GPUPatterns.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/LLVMGPU/Utils/LLVMGPUUtils.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "mlir/Conversion/VectorToGPU/VectorToGPU.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/PassDetail.h b/compiler/src/iree/compiler/Codegen/LLVMGPU/PassDetail.h
similarity index 69%
copy from compiler/src/iree/compiler/Codegen/PassDetail.h
copy to compiler/src/iree/compiler/Codegen/LLVMGPU/PassDetail.h
index fea75f0..70bf1d4 100644
--- a/compiler/src/iree/compiler/Codegen/PassDetail.h
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/PassDetail.h
@@ -1,11 +1,11 @@
-// Copyright 2021 The IREE Authors
+// Copyright 2023 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#ifndef IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
-#define IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#ifndef IREE_COMPILER_CODEGEN_LLVMGPU_PASS_DETAIL_H_
+#define IREE_COMPILER_CODEGEN_LLVMGPU_PASS_DETAIL_H_
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
@@ -18,9 +18,9 @@
namespace iree_compiler {
#define GEN_PASS_CLASSES
-#include "iree/compiler/Codegen/Passes.h.inc"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h.inc"
} // namespace iree_compiler
} // namespace mlir
-#endif // IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#endif // IREE_COMPILER_CODEGEN_LLVMGPU_PASS_DETAIL_H_
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.cpp
index b2a0fa0..e9eeeef 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.cpp
@@ -7,11 +7,10 @@
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
#include "iree-dialects/Dialect/LinalgTransform/Passes.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/Passes.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
#include "iree/compiler/Dialect/Util/Transforms/Passes.h"
@@ -591,5 +590,33 @@
});
}
+//===---------------------------------------------------------------------===//
+// Register LLVMGPU Passes
+//===---------------------------------------------------------------------===//
+
+namespace {
+#define GEN_PASS_REGISTRATION
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h.inc"
+} // namespace
+
+void registerCodegenLLVMGPUPasses() {
+ // Generated.
+ registerPasses();
+
+ static PassPipelineRegistration<> LinalgNVVMPipeline(
+ "iree-codegen-linalg-to-nvvm-pipeline",
+ "Runs the progressive lowering pipeline from Linalg to NVVM",
+ [](OpPassManager &passManager) {
+ buildLLVMGPUTransformPassPipeline(passManager, false);
+ });
+
+ static PassPipelineRegistration<> LinalgROCDLPipeline(
+ "iree-codegen-linalg-to-rocdl-pipeline",
+ "Runs the progressive lowering pipeline from Linalg to ROCDL",
+ [](OpPassManager &passManager) {
+ buildLLVMGPUTransformPassPipeline(passManager, true);
+ });
+}
+
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.h
similarity index 94%
rename from compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h
rename to compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.h
index ecab5de..d1cab28 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.h
@@ -60,10 +60,6 @@
/// Performs the final conversion to ROCDL+LLVM dialect.
std::unique_ptr<OperationPass<ModuleOp>> createConvertToROCDLPass();
-// Extract address computations (including the ones with GPU instructions) into
-// their own separate instructions.
-std::unique_ptr<Pass> createExtractAddressComputationGPUPass();
-
/// Cast address space to generic in CallOp and FuncOp
std::unique_ptr<OperationPass<ModuleOp>>
createLLVMGPUCastAddressSpaceFunction();
@@ -110,6 +106,12 @@
IREE::Codegen::TranslationInfoAttr translationInfo,
ArrayRef<int64_t> workgroupSize);
+//----------------------------------------------------------------------------//
+// Register LLVMGPU Passes
+//----------------------------------------------------------------------------//
+
+void registerCodegenLLVMGPUPasses();
+
//------------------------------------------------------------------------------
// Test passes
//------------------------------------------------------------------------------
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.td b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.td
new file mode 100644
index 0000000..01f72d6
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/Passes.td
@@ -0,0 +1,103 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef IREE_CODEGEN_LLVMGPU_PASSES
+#define IREE_CODEGEN_LLVMGPU_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+//------------------------------------------------------------------------------
+// LLVMGPU Passes (keep alphabetical)
+//------------------------------------------------------------------------------
+
+// TODO: Bring the argument in line with the names used elsewhere.
+def ConvertToNVVM :
+ Pass<"iree-convert-to-nvvm", "ModuleOp"> {
+ let summary = "Perform final conversion from builtin/GPU/HAL/standard dialect to LLVM "
+ "and NVVM dialects";
+ let constructor = "mlir::iree_compiler::createConvertToNVVMPass()";
+}
+
+// TODO: Bring the argument in line with the names used elsewhere.
+def ConvertToROCDL :
+ Pass<"iree-convert-to-rocdl", "ModuleOp"> {
+ let summary = "Perform final conversion from builtin/GPU/HAL/standard dialect to LLVM "
+ "and ROCDL dialects";
+ let constructor = "mlir::iree_compiler::createConvertToROCDLPass()";
+}
+
+def ExtractAddressComputationGPU: Pass<"extract-address-computation-gpu"> {
+ let summary = "Extract address computations from memory accesses";
+ let description = [{
+ This pass is similar to `extract-address-computation` except it also
+ supports memory accesses that are specific to GPUs.
+ }];
+ let constructor = "mlir::iree_compiler::createExtractAddressComputationGPUPass()";
+ let dependentDialects = [
+ "memref::MemRefDialect", "nvgpu::NVGPUDialect", "affine::AffineDialect"
+ ];
+}
+
+def LLVMGPUCastAddressSpaceFunction :
+ Pass<"iree-llvmgpu-cast-address-space-function", "ModuleOp"> {
+ let summary = "Pass to cast";
+ let constructor = "mlir::iree_compiler::createLLVMGPUCastAddressSpaceFunction()";
+}
+
+// TODO: Bring the argument in line with the names used elsewhere.
+def LLVMGPULowerExecutableTarget :
+ Pass<"iree-llvmgpu-lower-executable-target", "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
+ let summary = "Perform lowering of executable target using one of the IREE::HAL::DispatchLoweringPassPipeline";
+ let constructor = "mlir::iree_compiler::createLLVMGPULowerExecutableTargetPass()";
+}
+
+def LLVMGPUPackSharedMemoryAlloc :
+ Pass<"iree-llvmgpu-pack-shared-memory-alloc", "func::FuncOp"> {
+ let summary = "Pass pack shared memory allocation in order to reduce memory usage.";
+ let constructor = "mlir::iree_compiler::createLLVMGPUPackSharedMemoryAlloc()";
+}
+
+def LLVMGPUTensorCoreVectorization :
+ Pass<"iree-llvmgpu-tensorcore-vectorization", "func::FuncOp"> {
+ let summary = "Pass to convert linalg into Vector and transform it to a form that can be lowered to GPU MMA ops";
+ let constructor = "mlir::iree_compiler::createLLVMGPUTensorCoreVectorizationPass()";
+}
+
+def LLVMGPUTensorPad :
+ Pass<"iree-llvmgpu-tensor-pad", "func::FuncOp"> {
+ let summary = "Pass to pad out tensors up to static dimensions.";
+ let constructor = "mlir::iree_compiler::createLLVMGPUTensorPadPass()";
+}
+
+def LLVMGPUTileAndDistribute :
+ Pass<"iree-llvmgpu-tile-and-distribute", "func::FuncOp"> {
+ let summary = "Pass to tile and distribute linalg ops within a workgroup.";
+ let constructor = "mlir::iree_compiler::createLLVMGPUTileAndDistribute()";
+}
+
+def LLVMGPUVectorLowering :
+ Pass<"iree-llvmgpu-vector-lowering", "func::FuncOp"> {
+ let summary = "Pass to lower Vector ops before conversion to LLVM.";
+ let constructor = "mlir::iree_compiler::createLLVMGPUVectorLoweringPass()";
+}
+
+def LLVMGPUVectorToGPU :
+ Pass<"iree-llvmgpu-vector-to-gpu", "func::FuncOp"> {
+ let summary = "Pass to convert vector to gpu.";
+ let constructor = "mlir::iree_compiler::createLLVMGPUVectorToGPU()";
+}
+
+//------------------------------------------------------------------------------
+// Test Passes
+//------------------------------------------------------------------------------
+
+def TestLLVMGPUScalarizeMathOp :
+ Pass<"iree-test-llvmgpu-legalize-ops", "ModuleOp"> {
+ let summary = "Test pass for several legalization patterns.";
+ let constructor = "mlir::iree_compiler::createTestLLVMGPULegalizePass()";
+}
+
+#endif // IREE_CODEGEN_LLVMGPU_PASSES
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/BUILD.bazel b/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/BUILD.bazel
index 6b0c8e7..7f68c8d 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/BUILD.bazel
@@ -57,7 +57,7 @@
],
deps = [
":LLVMGPUExtensionsOpGen",
- "//compiler/src/iree/compiler/Codegen/Common:CommonPasses",
+ "//compiler/src/iree/compiler/Codegen/Common",
"//compiler/src/iree/compiler/Codegen/Common/GPU:CommonGPUPasses",
"//compiler/src/iree/compiler/Codegen/LLVMGPU/Utils",
"//compiler/src/iree/compiler/Codegen/Utils",
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/CMakeLists.txt
index 82bea7a..8491b58 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/CMakeLists.txt
@@ -57,7 +57,7 @@
MLIRVectorToGPU
MLIRVectorTransforms
MLIRViewLikeInterface
- iree::compiler::Codegen::Common::CommonPasses
+ iree::compiler::Codegen::Common
iree::compiler::Codegen::Common::GPU::CommonGPUPasses
iree::compiler::Codegen::LLVMGPU::Utils
iree::compiler::Codegen::Utils
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/LLVMGPUExtensions.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/LLVMGPUExtensions.cpp
index 2175617..2ea8a69 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/LLVMGPUExtensions.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/TransformExtensions/LLVMGPUExtensions.cpp
@@ -8,7 +8,7 @@
#include "iree-dialects/Dialect/LinalgTransform/SimplePatternRewriter.h"
#include "iree-dialects/Dialect/LinalgTransform/StructuredTransformOpsExt.h"
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
#include "iree/compiler/Codegen/LLVMGPU/Utils/LLVMGPUUtils.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/LLVMGPU/Verifiers.cpp b/compiler/src/iree/compiler/Codegen/LLVMGPU/Verifiers.cpp
index 315f4f2..15efa82 100644
--- a/compiler/src/iree/compiler/Codegen/LLVMGPU/Verifiers.cpp
+++ b/compiler/src/iree/compiler/Codegen/LLVMGPU/Verifiers.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/PassDetail.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "mlir/Dialect/Linalg/Passes.h"
diff --git a/compiler/src/iree/compiler/Codegen/Passes.cpp b/compiler/src/iree/compiler/Codegen/Passes.cpp
index 102984a..293ea73 100644
--- a/compiler/src/iree/compiler/Codegen/Passes.cpp
+++ b/compiler/src/iree/compiler/Codegen/Passes.cpp
@@ -4,85 +4,31 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
-
-#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
#include "iree/compiler/Codegen/Passes.h"
#include "mlir/Pass/PassManager.h"
//===---------------------------------------------------------------------===//
// Include pass headers per target device
//===---------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
-#include "iree/compiler/Codegen/VMVX/VMVXPasses.h"
-#include "iree/compiler/Codegen/WGSL/WGSLPasses.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
+#include "iree/compiler/Codegen/VMVX/Passes.h"
+#include "iree/compiler/Codegen/WGSL/Passes.h"
namespace mlir {
namespace iree_compiler {
-namespace {
-#define GEN_PASS_REGISTRATION
-#include "iree/compiler/Codegen/Passes.h.inc"
-} // namespace
-
void registerCodegenPasses() {
// Generated.
- registerPasses();
-
- static PassPipelineRegistration<> LinalgLLVMPipeline(
- "iree-codegen-linalg-to-llvm-pipeline",
- "Runs the progressive lowering pipeline from Linalg to LLVM",
- [](OpPassManager &passManager) {
- buildLLVMCPUCodegenPassPipeline(passManager);
- });
-
- static PassPipelineRegistration<> LinalgNVVMPipeline(
- "iree-codegen-linalg-to-nvvm-pipeline",
- "Runs the progressive lowering pipeline from Linalg to NVVM",
- [](OpPassManager &passManager) {
- buildLLVMGPUTransformPassPipeline(passManager, false);
- });
-
- static PassPipelineRegistration<> LinalgROCDLPipeline(
- "iree-codegen-linalg-to-rocdl-pipeline",
- "Runs the progressive lowering pipeline from Linalg to ROCDL",
- [](OpPassManager &passManager) {
- buildLLVMGPUTransformPassPipeline(passManager, true);
- });
-
- static PassPipelineRegistration<> LinalgSPIRVPipeline(
- "iree-codegen-linalg-to-spirv-pipeline",
- "Runs the progressive lowering pipeline from linalg to SPIR-V",
- [](OpPassManager &passManager) {
- buildSPIRVCodegenPassPipeline(passManager, /*enableFastMath=*/false);
- });
-
- static PassPipelineRegistration<> LLVMCPULinkingPipeline(
- "iree-codegen-llvmcpu-linking-pipeline",
- "Runs the LLVMCPU HAL executable linking pipeline",
- [](OpPassManager &passManager) {
- buildLLVMCPULinkingPassPipeline(passManager);
- });
-
- static PassPipelineRegistration<> VMVXLinkingPipeline(
- "iree-codegen-vmvx-linking-pipeline",
- "Runs the VMVX HAL executable linking pipeline",
- [](OpPassManager &passManager) {
- buildVMVXLinkingPassPipeline(passManager);
- });
-}
-
-void addCommonTargetExecutablePreprocessingPasses(OpPassManager &passManager) {
- passManager.addNestedPass<func::FuncOp>(createTypePropagationPass());
- passManager.addPass(createBubbleUpOrdinalOpsPass());
- passManager.addPass(createBufferizeCopyOnlyDispatchesPass());
- passManager.addNestedPass<func::FuncOp>(
- IREE::LinalgExt::createDecomposeSoftmaxPass());
- // Temporary solution to avoid large allocations due to softmax lowering.
- passManager.addNestedPass<func::FuncOp>(createRematerializeParallelOpsPass());
+ registerCodegenCommonPasses();
+ registerCodegenCommonGPUPasses();
+ registerCodegenLLVMCPUPasses();
+ registerCodegenLLVMGPUPasses();
+ registerCodegenSPIRVPasses();
+ registerCodegenVMVXPasses();
+ registerCodegenWGSLPasses();
}
} // namespace iree_compiler
diff --git a/compiler/src/iree/compiler/Codegen/Passes.td b/compiler/src/iree/compiler/Codegen/Passes.td
deleted file mode 100644
index 08f6c79..0000000
--- a/compiler/src/iree/compiler/Codegen/Passes.td
+++ /dev/null
@@ -1,959 +0,0 @@
-// Copyright 2021 The IREE Authors
-//
-// Licensed under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-
-#ifndef IREE_CODEGEN_PASSES
-#define IREE_CODEGEN_PASSES
-
-include "mlir/Pass/PassBase.td"
-
-//------------------------------------------------------------------------------
-// Common/misc passes
-//------------------------------------------------------------------------------
-
-def BubbleUpOrdinalOps : Pass<"iree-codegen-bubble-up-ordinal-ops", ""> {
- let summary = "Bubbles op ordinal ops to allow for workgroup count computation";
- let constructor = "mlir::iree_compiler::createBubbleUpOrdinalOpsPass()";
-}
-
-def CleanupBufferAllocView :
- Pass<"iree-codegen-cleanup-buffer-alloc-view", "func::FuncOp"> {
- let summary =
- "Performs cleanups over HAL interface/buffer allocation/view operations";
- let constructor = "mlir::iree_compiler::createCleanupBufferAllocViewPass()";
-}
-
-def ConvertToDestinationPassingStyle :
- Pass<"iree-codegen-convert-to-destination-passing-style", "func::FuncOp"> {
- let summary =
- "Transforms the code to make the dispatch use destination-passing style";
- let constructor = "mlir::iree_compiler::createConvertToDestinationPassingStylePass()";
- let options = [
- Option<"useWARForCooperativeMatrixCodegen", "use-war-for-cooperative-matrix-codegen",
- "bool", /*default=*/"false",
- "WAR for failure in Cooperative matrix codegen pipelines. See #10648.">
- ];
-}
-
-def DecomposeLinalgGeneric :
- Pass<"iree-codegen-decompose-linalg-generic", ""> {
- let summary = "Decomposes linalg generic ops into individual ops";
- let description = [{
- It is sometimes advantageous to operate on generic ops which contain
- at most one non-yield body operation. This is most often the case when
- needing to materialize individual ops (which some backends require).
- Note that this is often an extreme pessimization unless if part of a
- lowering flow which was designed for it.
-
- Operates on tensor based linalg ops.
- }];
- let constructor = "mlir::iree_compiler::createDecomposeLinalgGenericPass()";
-}
-
-def DecomposeConvolutionToLowerDimOps :
- Pass<"iree-codegen-decompose-convolution-to-lower-dim-ops", ""> {
- let summary = "Decomposes linalg convolution ops to lower dim ops";
- let constructor =
- "mlir::iree_compiler::createDecomposeConvolutionToLowerDimOpsPass()";
-}
-
-def DecomposeAffineOps: Pass<"decompose-affine-ops"> {
- let summary = "Decompose `affine.apply` operations into sub `affine.apply`";
- let description = [{
- Decompose `affine.apply` operations into sub `affine.apply` where each
- sub expression references values that are defined in the same loop scope.
- The sub expression are then stitched back together following the loop
- nest order.
-
- The goal of this pass is to break down `affine.apply` expressions such
- that the resulting sub expressions can be hoisted out in their respective
- loop.
-
- E.g., Let's say we have
- ```mlir
- %res = affine.apply
- affine_map<()[s0, s1, s2] -> (s0 * 1024 + s1 * 32 + s2)>()
- [%loopVariant, %inv1, %inv2]
- ```
- Where `%inv1` and `%inv2` are loop invariant and `%loopVariant` is not.
- This will produce the following subexpressions:
- ```mlir
- // Loop invariant computations first.
- %inv1x32 =
- affine.apply affine_map<()[s0] -> (s0 * 32)>()[%inv1]
- %inv1x32_plus_inv2 =
- affine.apply affine_map<()[s0, s1] -> (s0 + s1)>()[%inv1x32, %inv2]
- // Loop variant computation next.
- %loopVariantx1024 =
- affine.apply affine_map<()[s0] -> (s0 * 1024)>()[%loopVariant]
- // Compose things back together.
- %res =
- affine.apply affine_map<()[s0, s1] -> (s0 + s1)>()
- [%loopVariant, %inv1x32_plus_inv2]
- ```
- Now the sequence of instructions leading to and including
- `%inv1x32_plus_inv2` can be hoisted out of the loop.
-
- This pass requires `scf.for` structures to still be around otherwise
- the break down will be meaningless.
-
- Note: The decomposition performed by this pass will be undone by
- canonicalization. Make sure to lower the resulting ops before that.
- }];
- let constructor = "mlir::iree_compiler::createDecomposeAffineOpsPass()";
- let dependentDialects = [
- "affine::AffineDialect"
- ];
-}
-
-def ConvertBf16ToUInt16Buffers :
- Pass<"iree-convert-bf16-to-uint16-buffers", "ModuleOp"> {
- let summary = "Convert bf16 buffers to uint16 equivalents";
- let constructor = "mlir::iree_compiler::createConvertBf16ToUInt16BuffersPass()";
-}
-
-def ConvertBf16ArithToF32 : Pass<"iree-convert-bf16-arith-to-f32", "ModuleOp"> {
- let summary = "Convert bf16 arithmetic operations to f32";
- let constructor = "mlir::iree_compiler::createConvertBf16ArithToF32Pass()";
-}
-
-def ExtractAddressComputation: Pass<"extract-address-computation"> {
- let summary = "Extract address computations from memory accesses";
- let description = [{
- Extract the address computation from the instructions with memory
- accesses such that these memory accesses use only a base pointer.
-
- For instance,
- ```mlir
- memref.load %base[%off0, ...]
- ```
-
- Will be rewritten in:
- ```mlir
- %new_base = memref.subview %base[%off0,...][1,...][1,...]
- memref.load %new_base[%c0,...]
- ```
- }];
- let constructor = "mlir::iree_compiler::createExtractAddressComputationPass()";
- let dependentDialects = [
- "memref::MemRefDialect"
- ];
-}
-
-def ExtractAddressComputationGPU: Pass<"extract-address-computation-gpu"> {
- let summary = "Extract address computations from memory accesses";
- let description = [{
- This pass is similar to `extract-address-computation` except it also
- supports memory accesses that are specific to GPUs.
- }];
- let constructor = "mlir::iree_compiler::createExtractAddressComputationGPUPass()";
- let dependentDialects = [
- "memref::MemRefDialect", "nvgpu::NVGPUDialect", "affine::AffineDialect"
- ];
-}
-
-def FlattenMemRefSubspan :
- Pass<"iree-codegen-flatten-memref-subspan", "ModuleOp"> {
- let summary =
- "Flatten n-D MemRef subspan ops to 1-D ones and fold byte offsets";
- let constructor = "mlir::iree_compiler::createFlattenMemRefSubspanPass()";
-}
-
-def FoldAffineMinInDistributedLoops :
- Pass<"iree-codegen-fold-affinemin-in-distributed-loops", "func::FuncOp"> {
- let summary = "Fold `affine.min` ops in distributed loops";
- let constructor = "mlir::iree_compiler::createFoldAffineMinInDistributedLoopsPass()";
-}
-
-def FoldTensorExtractOp :
- Pass<"iree-codegen-fold-tensor-extract-op", ""> {
- let summary = "Fold `tensor.extract` operations prior to lowering to LLVM";
- let constructor = "mlir::iree_compiler::createFoldTensorExtractOpPass()";
-}
-
-def ForOpCanonicalization :
- Pass<"iree-codegen-canonicalize-scf-for", "func::FuncOp"> {
- let summary =
- "Adhoc canonicalization of selected loop-carried values/dependencies for scf.for ops";
- let constructor = "mlir::iree_compiler::createForOpCanonicalizationPass()";
-}
-
-def BufferizeCopyOnlyDispatches :
- Pass<"iree-codegen-bufferize-copy-only-dispatches", "ModuleOp"> {
- let summary =
- "Bufferize dispatches that copy to/from interfaces to convert to a linalg.copy op";
- let constructor = "mlir::iree_compiler::createBufferizeCopyOnlyDispatchesPass()";
-}
-
-def EliminateEmptyTensors :
- Pass<"iree-eliminate-empty-tensors", "ModuleOp"> {
- let summary = "Eliminate tensor.empty ops to avoid buffer allocations";
- let constructor = "mlir::iree_compiler::createEliminateEmptyTensorsPass()";
-}
-
-def HoistStaticallyBoundAllocations :
- Pass<"iree-hoist-statically-bound-allocations", "func::FuncOp"> {
- let summary = "Hoist statically bound alloca ops to the entry block of functions";
- let constructor = "mlir::iree_compiler::createHoistStaticallyBoundAllocationsPass()";
-}
-
-def IREEComprehensiveBufferize :
- Pass<"iree-codegen-iree-comprehensive-bufferize", "ModuleOp"> {
- let summary = "Convert from to Linalg ops on tensors to buffers";
- let constructor = "mlir::iree_compiler::createIREEComprehensiveBufferizePass()";
- let options = [
- Option<"testAnalysisOnly", "test-analysis-only", "bool",
- /*default=*/"false",
- "Only runs inplaceability analysis (for testing purposes only)">,
- Option<"printConflicts", "print-conflicts", "bool",
- /*default=*/"false",
- "Annotates IR with RaW conflicts. Requires test-analysis-only.">,
- ];
-}
-
-def IREEExpandStridedMetadata :
- Pass<"iree-codegen-expand-strided-metadata", ""> {
- let summary = "Resolve memref.extract_strided_metadata operations";
- let constructor = "mlir::iree_compiler::createIREEExpandStridedMetadataPass()";
- let options = [
- Option<"allowUnresolved", "allow-unresolved", "bool", /*default=*/"false",
- "Allow unresolved strided metadata op (for testing)">,
- ];
-}
-
-def LowerUKernelOpsToCalls :
- Pass<"iree-codegen-lower-ukernel-ops-to-calls", "ModuleOp"> {
- let summary = "Lower micro-kernel wrapper ops into function calls";
- let constructor = "mlir::iree_compiler::createLowerUKernelOpsToCallsPass()";
-}
-
-def OptimizeVectorTransfer :
- Pass<"iree-codegen-optimize-vector-transfer", "func::FuncOp"> {
- let summary =
- "Run optimization transformations on vector transfer operations";
- let constructor = "mlir::iree_compiler::createOptimizeVectorTransferPass()";
- let options = [
- Option<"optionFlatten", "flatten", "bool", "false",
- "Flatten the vector type of vector transfers where possible (contiguous row-major data).">,
- Option<"optionDropUnitDims", "drop-unit-dims", "bool", /*default=*/"true",
- "Drop unit dims in vector transfers where possible (might generate vector.shape_cast).">,
- ];
- let dependentDialects = [
- "memref::MemRefDialect"
- ];
-}
-
-def SplitFullPartialTransfer :
- Pass<"iree-codegen-split-full-partial-transfer", "func::FuncOp"> {
- let summary =
- "Split a vector.transfer operation into an in-bounds (i.e., no "
- "out-of-bounds masking) fastpath and a slowpath.";
- let constructor = "mlir::iree_compiler::createSplitFullPartialTransferPass()";
- let options = [
- Option<"splitVectorTransfersTo", "split-transfers", "std::string",
- /*default=*/"",
- [{Split vector transfers between slow (masked) and fast "
- "(unmasked) variants. Possible options are:\n"
- "\tnone [default]: keep unsplit vector.transfer and pay the price\n"
- "\tlinalg-copy: use linalg.fill + linalg.generic for the slow path\n"
- "\tvector-transfers: use extra small unmasked vector.transfers for"
- " the slow path\n}]>,
- ];
-}
-
-def TestExecutablePreprocessing :
- Pass<"iree-codegen-test-executable-preprocessing", ""> {
- let summary = "Tests iree-hal-preprocess-executables-with behavior.";
- let constructor = "mlir::iree_compiler::createTestExecutablePreprocessingPass()";
-}
-
-def TestPartitionableLoopsInterface :
- Pass<"iree-codegen-test-partitionable-loops-interface", ""> {
- let summary = "Test the PartitionableLoopsInterface";
- let constructor = "mlir::iree_compiler::createTestPartitionableLoopsInterfacePass()";
-}
-
-def TileAndDistributeToWorkgroups :
- Pass<"iree-codegen-tile-and-distribute-to-workgroups", "IREE::HAL::ExecutableVariantOp"> {
- let summary = "Tile and distribute operations to workgroups";
- let constructor = "mlir::iree_compiler::createTileAndDistributeToWorkgroupsPass()";
- let options = [
- Option<"maxWorkgroupParallelDims", "max-workgroup-parallel-dims", "int32_t",
- /*default=*/ "",
- "Maximum number of dims to distribute workgroups across.">,
- Option<"distributionMethod", "distribution-method", "int32_t",
- /*default=*/ "0",
- "Pick the distribution method">
- ];
-}
-
-def WorkgroupSpecialization :
- Pass<"iree-codegen-workgroup-specialization", "func::FuncOp"> {
- let summary = "Specialize workgroup distribution loops";
- let constructor = "mlir::iree_compiler::createWorkgroupSpecializationPass()";
-}
-
-def TypePropagation :
- Pass<"iree-codegen-type-propagation", "func::FuncOp"> {
- let summary = "Propogate the type of tensor to avoid load/stores of illegal bit widths";
- let constructor = "mlir::iree_compiler::createTypePropagationPass()";
-}
-
-def RemoveSingleIterationLoop :
- Pass<"iree-codegen-remove-single-iteration-loop", "func::FuncOp"> {
- let summary = "Remove distributed loop with single iteration.";
- let constructor = "mlir::iree_compiler::createRemoveSingleIterationLoopPass()";
-}
-
-def TensorToVectorVectorizePad :
- Pass<"iree-codegen-vectorize-tensor-pad", "func::FuncOp"> {
- let summary = "Vectorize a very specific form of tensor.pad with "
- "control flows";
- let constructor =
- "mlir::iree_compiler::createVectorizePadPass()";
-}
-
-def DecomposePackUnPackOps :
- Pass<"iree-codegen-decompose-pack-unpack-ops", "func::FuncOp"> {
- let summary = "Decompose pack/unpack ops into vectorizable ops";
- let constructor = "mlir::iree_compiler::createDecomposePackUnPackOpsPass()";
- let options = [
- Option<"tileOuterToOne", "tile-outer-to-one", "bool", "false",
- "Always apply tiling to make outer dimension be ones">
- ];
-}
-
-def PolynomialApproximationPass :
- Pass<"iree-codegen-polynomial-approximation", ""> {
- let summary = "Convert math operations to their polynomial approximation";
- let constructor =
- "mlir::iree_compiler::createPolynomialApproximationPass()";
-}
-
-def MemrefCopyToLinalgPass :
- Pass<"iree-codegen-memrefcopy-to-linalg", "func::FuncOp"> {
- let summary = "Convert memref.copy to linalg op";
- let constructor =
- "mlir::iree_compiler::createMemrefCopyToLinalgPass()";
-}
-
-def GPUCheckResourceUsage :
- Pass<"iree-codegen-gpu-check-resource-usage", "ModuleOp"> {
- let summary = "Checks GPU specific resource usage constraints like shared memory limits";
- let constructor = "mlir::iree_compiler::createGPUCheckResourceUsagePass()";
-}
-
-def GPUDistribute :
- Pass<"iree-codegen-gpu-distribute", "func::FuncOp"> {
- let summary = "Pass to distribute scf.forall ops.";
- let constructor = "mlir::iree_compiler::createGPUDistribute()";
-}
-
-def GPUDistributeSharedMemoryCopy :
- Pass<"iree-codegen-gpu-distribute-shared-memory-copy", "func::FuncOp"> {
- let summary = "Pass to distribute shared memory copies to threads.";
- let constructor = "mlir::iree_compiler::createGPUDistributeSharedMemoryCopy()";
-}
-
-def GPUReduceBankConflicts :
- Pass<"iree-codegen-gpu-reduce-bank-conflicts", "func::FuncOp"> {
- let summary = "Pass to try to reduce the number of bank conflicts.";
- let constructor = "mlir::iree_compiler::createGPUReduceSharedMemoryBankConflicts()";
-}
-
-def GPUMultiBuffering :
- Pass<"iree-codegen-gpu-multi-buffering", "func::FuncOp"> {
- let summary = "Pass to do multi buffering.";
- let constructor = "mlir::iree_compiler::createGPUMultiBuffering()";
-}
-
-def GPUPipelining : Pass<"iree-codegen-gpu-pipelining", "func::FuncOp"> {
- let summary = "Pass to do software pipelining.";
- let constructor = "mlir::iree_compiler::createGPUPipeliningPass()";
- let options = [
- Option<"epiloguePeeling", "epilogue-peeling", "bool",
- /*default=*/"true",
- "Try to use un-peeling epilogue when false, peeled epilouge o.w.">,
- Option<"depth", "pipeline-depth", "int64_t",
- /*default=*/"2",
- "Number of stages ">,
- Option<"scheduleIndex", "schedule-index", "int64_t",
- /*default=*/"0",
- "Allows picking different schedule for the pipelining transformation.">,
- Option<"transformFileName", "transform-file-name", "std::string",
- /*default=*/"\"\"",
- "Optional filename containing a transform dialect specification to "
- "apply. If left empty, the IR is assumed to contain one top-level "
- "transform dialect operation somewhere in the module.">,
- ];
-}
-
-def WorkGroupSwizzle :
- Pass<"iree-workgroup-swizzle", "func::FuncOp"> {
- let summary = "swizzle the workgroup ids for better cache reuse";
- let constructor = "mlir::iree_compiler::createWorkGroupSwizzle()";
- let options = [
- Option<"logTile", "logTile", "unsigned",
- /*default=*/"0",
- "pass the tile value for unit testing">,
- ];
-}
-
-def PadDynamicAlloc :
- Pass<"iree-codegen-pad-dynamic-alloc", "func::FuncOp"> {
- let summary = "Pass to pad dynamic alloc into static one.";
- let constructor = "mlir::iree_compiler::createPadDynamicAlloc()";
-}
-
-def TransformDialectInterpreter :
- Pass<"iree-transform-dialect-interpreter"> {
- let summary = "Pass to apply transform dialect operations.";
- let constructor =
- "mlir::iree_compiler::createTransformDialectInterpreterPass()";
- let options = [
- Option<"transformFileName", "transform-file-name", "std::string",
- /*default=*/"\"\"",
- "Optional filename containing a transform dialect specification to "
- "apply. If left empty, the IR is assumed to contain one top-level "
- "transform dialect operation somewhere in the module.">,
- Option<"transformLibraryFileName",
- "transform-library-file-name",
- "std::string",
- /*default=*/"\"\"",
- "If non-empty, the name of the file containing definitions of "
- "external symbols referenced in the transform script. "
- "These definitions will be used to replace declarations.">,
- Option<"debugPayloadRootTag", "debug-payload-root-tag", "std::string",
- /*default=*/"\"\"",
- "Select the operation with 'transform.target_tag' attribute having "
- "the given value as payload IR root. This allows user control on "
- "what operation to transform in debug mode, without requiring "
- "intimate knowledge of the IREE nested pass pipeline.\\n"
- "If empty (normal operation mode), select the pass anchor "
- "operation in the IREE pipeline, as the payload IR root.">,
- Option<"debugTransformRootTag", "debug-transform-root-tag", "std::string",
- /*default=*/"\"\"",
- "Select the operation with 'transform.target_tag' attribute having "
- "the given value as container IR for top-level transform ops. This "
- "allows user control on what transformation to apply in debug "
- "mode, without requiring intimate knowledge of the IREE nested "
- "pass pipeline.\\n"
- "If empty (normal operation mode), select the container of the "
- "top-level transform op.">
- ];
-}
-
-def GPUVectorization :
- Pass<"iree-codegen-gpu-vectorization", "func::FuncOp"> {
- let summary = "Pass to convert linalg into Vector.";
- let constructor = "mlir::iree_compiler::createGPUVectorizationPass()";
- let options = [
- Option<"generateContract", "generate-contract", "bool",
- /*default=*/"true",
- "Try to convert reduction to vector.contract.">,
- Option<"maxVectorSize", "max-vector-size", "int64_t",
- /*default=*/"4096",
- "Max vector size allowed to avoid creating large vectors.">
- ];
-}
-
-def GPUTensorAlloc :
- Pass<"iree-codegen-gpu-tensor-alloc", "func::FuncOp"> {
- let summary = "Pass to tile reduction dimensions and create allocations for "
- "some tensor values to use GPU shared memory";
- let constructor = "mlir::iree_compiler::createGPUTensorAlloc()";
-}
-
-def GPUTensorTile :
- Pass<"iree-codegen-gpu-tensor-tile", "func::FuncOp"> {
- let summary = "Pass to tile tensor (linalg) ops within a GPU workgroup";
- let constructor = "mlir::iree_compiler::createGPUTensorTile()";
-}
-
-def GPUTileReduction :
- Pass<"iree-codegen-gpu-tile-reduction", "func::FuncOp"> {
- let summary = "Pass to tile linalg reduction dimensions.";
- let constructor = "mlir::iree_compiler::createGPUTileReductionPass()";
-}
-
-def VectorReduceToGPU :
- Pass<"iree-codegen-reduction-to-gpu", "func::FuncOp"> {
- let summary = "Convert vector reduction to gpu ops.";
- let constructor = "mlir::iree_compiler::createConvertVectorReductionToGPUPass()";
-}
-
-def FuseTensorPadWithConsumer :
- Pass<"iree-codegen-fuse-tensor-pad-with-consumer", "func::FuncOp"> {
- let summary = "Fuse tensor.pad op into its consumer op's tiled loop nest";
- let constructor = "mlir::iree_compiler::createFuseTensorPadWithConsumerPass()";
-}
-
-def ConcretizePadResultShape :
- Pass<"iree-codegen-concretize-pad-result-shape", "func::FuncOp"> {
- let summary =
- "Concretizes tensor.pad op's result shape if its source op"
- "implements OffsetSizeAndStrideOpInterface.";
- let constructor = "mlir::iree_compiler::createConcretizePadResultShapePass()";
-}
-
-def EraseHALDescriptorTypeFromMemRef :
- Pass<"iree-codegen-erase-hal-descriptor-type-from-memref", "func::FuncOp"> {
- let summary = "Erase #hal.descriptor_type from MemRef memory space";
- let constructor =
- "mlir::iree_compiler::createEraseHALDescriptorTypeFromMemRefPass()";
-}
-
-def RematerializeParallelOps :
- Pass<"iree-codegen-rematerialize-parallel-ops", "func::FuncOp"> {
- let summary = "Pass to rematerialize and merge parallel ops to avoid creating temporary allocs.";
- let constructor = "mlir::iree_compiler::createRematerializeParallelOpsPass()";
-}
-
-def InstrumentMemoryAccesses :
- Pass<"iree-codegen-instrument-memory-accesses", "func::FuncOp"> {
- let summary = "Instruments memory reads and writes for address tracking when dispatch instrumentation is enabled.";
- let constructor = "mlir::iree_compiler::createInstrumentMemoryAccessesPass()";
-}
-
-//------------------------------------------------------------------------------
-// LLVMCPU
-//------------------------------------------------------------------------------
-
-def VerifyLinalgTransformLegality :
- Pass<"iree-llvmcpu-verify-linalg-transform-legality", "ModuleOp"> {
- let summary = "Verify that only supported IR constructs are passed to the compiler.";
- let constructor = "mlir::iree_compiler::createVerifyLinalgTransformLegalityPass()";
-}
-
-def LLVMCPUTile :
- Pass<"iree-llvmcpu-tile", "func::FuncOp"> {
- let summary = "Pass to tile TilingInterface operations.";
- let constructor =
- "mlir::iree_compiler::createLLVMCPUTilePass()";
- let options = [
- Option<"tilingLevel", "tiling-level", "int64_t", /*default=*/"-1",
- "Use default tiling level used to retrieve the configuration from lowering_config">
- ];
-}
-
-def LLVMCPUTileAndFuse :
- Pass<"iree-llvmcpu-tile-and-fuse", "func::FuncOp"> {
- let summary = "Pass to tile and fuse TilingInterface operations.";
- let constructor =
- "mlir::iree_compiler::createLLVMCPUTileAndFusePass()";
- let options = [
- Option<"tilingLevel", "tiling-level", "int64_t", /*default=*/"-1",
- "Use default tiling level used to retrieve the configuration from lowering_config">
- ];
-}
-
-def LLVMCPUTensorPad :
- Pass<"iree-llvmcpu-tensor-pad", "func::FuncOp"> {
- let summary = "Pass to pad operations on tensors in top-down order.";
- let constructor =
- "mlir::iree_compiler::createLLVMCPUTensorPadPass()";
-}
-
-def LLVMCPUPeel :
- Pass<"iree-llvmcpu-peel", "func::FuncOp"> {
- let summary = "Pass to perform peeling on non-distributed loops.";
- let constructor =
- "mlir::iree_compiler::createLLVMCPUPeelPass()";
-}
-
-def LLVMCPUSplitReduction : Pass<"iree-llvmcpu-split-reduction", "func::FuncOp"> {
- let summary = "Pass to splitReduce linalg operations.";
- let constructor = "mlir::iree_compiler::createLLVMCPUSplitReductionPass()";
- let options = [
- Option<"enableFpReductionReordering", "enable-fp-reduction-reordering",
- "bool", /*default=*/"false",
- "Flag to enable reduction reordering on floating points.">,
- ];
-}
-
-def LLVMCPUVectorization :
- Pass<"iree-llvmcpu-vectorization", "func::FuncOp"> {
- let summary = "Pass to perform vectorization on tensor/linalg ops.";
- let options = [
- Option<"enableVectorMasking", "enable-vector-masking", "bool",/*default=*/"false",
- "Enable vector masking during vectorization.">,
- Option<"vectorizePadding", "vectorize-padding", "bool", /*default=*/"false",
- "Rewrite all tensor.pad ops in the function to vector form.">,
- Option<"vectorizeGatherAccesses", "vectorize-gather-accesses", "bool", /*default=*/"false",
- "Enable vectorizaiton of operations that may generate vector.gather operations.">,
- ];
- let constructor =
- "mlir::iree_compiler::createLLVMCPUVectorizationPass()";
-}
-
-def LLVMCPUVectorLowering :
- Pass<"iree-llvmcpu-vector-lowering", "func::FuncOp"> {
- let summary = "Pass to lower Vector ops before conversion to LLVM.";
- let options = [
- Option<"splitVectorTransfersTo", "split-transfers", "std::string",
- /*default=*/"",
- [{Split vector transfers between slow (masked) and fast "
- "(unmasked) variants. Possible options are:\n"
- "\tnone [default]: keep unsplit vector.transfer and pay the price\n"
- "\tlinalg-copy: use linalg.fill + linalg.generic for the slow path\n"
- "\tvector-transfers: use extra small unmasked vector.transfers for"
- " the slow path\n}]>,
- Option<"lowerVectorTransposeToAVX2", "lower-vector-transpose-to-avx2", "bool",
- /*default=*/"false",
- "Add specific transpose to avx2 lowering patterns.">,
- ];
- let constructor =
- "mlir::iree_compiler::createLLVMCPUVectorLoweringPass()";
-}
-
-def ConvertToLLVM :
- Pass<"iree-convert-to-llvm", "ModuleOp"> {
- let summary =
- "Perform final conversion from Linalg/HAL/Shape/Vector/Standard to LLVMIR dialect";
- let constructor = "mlir::iree_compiler::createConvertToLLVMPass()";
- let options = [
- Option<"reassociateFpReductions", "reassociateFpReductions", "bool",
- /*default=*/"false",
- "Specifies if FP add and mult reductions can be reordered">,
- ];
-}
-
-def LLVMCPUEmitVectorizationRemarks :
- Pass<"iree-llvmcpu-emit-vectorization-remarks", "func::FuncOp"> {
- let summary = "Emit vectorization remarks on Linalg ops";
- let constructor =
- "mlir::iree_compiler::createLLVMCPUEmitVectorizationRemarksPass()";
-}
-
-def LLVMCPUCheckIRBeforeLLVMConversion :
- Pass<"iree-llvmcpu-check-ir-before-llvm-conversion", "ModuleOp"> {
- let summary = "Checks CPU backend specific IR constraints (like no allocas)";
- let constructor = "mlir::iree_compiler::createLLVMCPUCheckIRBeforeLLVMConversionPass()";
-}
-
-def LLVMCPULowerExecutableTarget :
- Pass<"iree-llvmcpu-lower-executable-target",
- "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
- let summary =
- "Lower executable target using an IREE::HAL::DispatchLoweringPassPipeline";
- let constructor =
- "mlir::iree_compiler::createLLVMCPULowerExecutableTargetPass()";
-}
-
-def ExpandArithF16ToF32 :
- Pass<"iree-llvmcpu-expand-f16-op-to-f32", ""> {
- let summary =
- "Preform f16 opertaions by expanding them to f32.";
- let constructor =
- "mlir::iree_compiler::createExpandF16OpToF32Pass()";
-}
-
-def LLVMCPULowerToUKernels :
- Pass<"iree-llvmcpu-lower-to-ukernels", ""> {
- let summary =
- "Separate out parts of the IR that lower to a micro-kernel";
- let constructor =
- "mlir::iree_compiler::createLLVMCPULowerToUKernelsPass()";
-}
-
-def LLVMCPUMaterializeEncoding :
- Pass<"iree-llvmcpu-materialize-encoding", "func::FuncOp"> {
- let summary = "Materialize the encoding for tensor as specified by the backend";
- let constructor = "mlir::iree_compiler::createLLVMCPUMaterializeEncodingPass()";
-}
-
-def LLVMCPUSynchronizeSymbolVisibility :
- Pass<"iree-llvmcpu-synchronize-symbol-visibility", "ModuleOp"> {
- let summary = "Synchronizes LLVM linkage with MLIR symbol visibility";
- let constructor = "mlir::iree_compiler::createLLVMCPUSynchronizeSymbolVisibilityPass()";
-}
-
-def LLVMCPUMmt4dVectorLowering
- : Pass<"iree-llvmcpu-mmt4d-vector-lowering", "func::FuncOp"> {
- let summary = "Apply vector lowering logic to vector ops";
- let constructor =
- "mlir::iree_compiler::createLLVMCPUMmt4dVectorLoweringPass()";
-}
-
-def LLVMCPUUnfuseFMAOps :
- Pass<"iree-llvmcpu-unfuse-fma-pass", "func::FuncOp"> {
- let summary = "Convert llvm.fma into unfused mulf and addf ops";
- let constructor = "mlir::iree_compiler::createLLVMCPUUnfuseFMAOpsPass()";
-}
-
-def VectorContractCustomKernels :
- Pass<"iree-llvmcpu-vector-contract-custom-kernels", "func::FuncOp"> {
- let summary = "Enable custom kernels (inline assembly or intrinsics) for some vector.contract ops";
- let constructor = "mlir::iree_compiler::createVectorContractCustomKernelsPass()";
-}
-
-def LLVMCPULinkExecutables :
- Pass<"iree-llvmcpu-link-executables", "mlir::ModuleOp"> {
- let summary = "Links LLVMCPU HAL executables within the top-level program module.";
- let constructor = "mlir::iree_compiler::createLLVMCPULinkExecutablesPass()";
-}
-
-def LLVMCPUAssignConstantOrdinals :
- Pass<"iree-llvmcpu-assign-constant-ordinals", "IREE::HAL::ExecutableVariantOp"> {
- let summary = "Assigns executable constant ordinals across all LLVMCPU variants.";
- let constructor = "mlir::iree_compiler::createLLVMCPUAssignConstantOrdinalsPass()";
-}
-
-def LLVMCPUAssignImportOrdinals :
- Pass<"iree-llvmcpu-assign-import-ordinals", "IREE::HAL::ExecutableVariantOp"> {
- let summary = "Assigns executable import ordinals across all LLVMCPU variants.";
- let constructor = "mlir::iree_compiler::createLLVMCPUAssignImportOrdinalsPass()";
-}
-
-//------------------------------------------------------------------------------
-// LLVMGPU
-//------------------------------------------------------------------------------
-
-// TODO: Bring the argument in line with the names used elsewhere.
-def ConvertToROCDL :
- Pass<"iree-convert-to-rocdl", "ModuleOp"> {
- let summary = "Perform final conversion from builtin/GPU/HAL/standard dialect to LLVM "
- "and ROCDL dialects";
- let constructor = "mlir::iree_compiler::createConvertToROCDLPass()";
-}
-
-// TODO: Bring the argument in line with the names used elsewhere.
-def ConvertToNVVM :
- Pass<"iree-convert-to-nvvm", "ModuleOp"> {
- let summary = "Perform final conversion from builtin/GPU/HAL/standard dialect to LLVM "
- "and NVVM dialects";
- let constructor = "mlir::iree_compiler::createConvertToNVVMPass()";
-}
-
-// TODO: Bring the argument in line with the names used elsewhere.
-def LLVMGPULowerExecutableTarget :
- Pass<"iree-llvmgpu-lower-executable-target", "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
- let summary = "Perform lowering of executable target using one of the IREE::HAL::DispatchLoweringPassPipeline";
- let constructor = "mlir::iree_compiler::createLLVMGPULowerExecutableTargetPass()";
-}
-
-def LLVMGPUCastAddressSpaceFunction :
- Pass<"iree-llvmgpu-cast-address-space-function", "ModuleOp"> {
- let summary = "Pass to cast";
- let constructor = "mlir::iree_compiler::createLLVMGPUCastAddressSpaceFunction()";
-}
-
-def LLVMGPUTileAndDistribute :
- Pass<"iree-llvmgpu-tile-and-distribute", "func::FuncOp"> {
- let summary = "Pass to tile and distribute linalg ops within a workgroup.";
- let constructor = "mlir::iree_compiler::createLLVMGPUTileAndDistribute()";
-}
-
-def LLVMGPUPackSharedMemoryAlloc :
- Pass<"iree-llvmgpu-pack-shared-memory-alloc", "func::FuncOp"> {
- let summary = "Pass pack shared memory allocation in order to reduce memory usage.";
- let constructor = "mlir::iree_compiler::createLLVMGPUPackSharedMemoryAlloc()";
-}
-
-def LLVMGPUTensorCoreVectorization :
- Pass<"iree-llvmgpu-tensorcore-vectorization", "func::FuncOp"> {
- let summary = "Pass to convert linalg into Vector and transform it to a form that can be lowered to GPU MMA ops";
- let constructor = "mlir::iree_compiler::createLLVMGPUTensorCoreVectorizationPass()";
-}
-
-def LLVMGPUVectorLowering :
- Pass<"iree-llvmgpu-vector-lowering", "func::FuncOp"> {
- let summary = "Pass to lower Vector ops before conversion to LLVM.";
- let constructor = "mlir::iree_compiler::createLLVMGPUVectorLoweringPass()";
-}
-
-def LLVMGPUVectorToGPU :
- Pass<"iree-llvmgpu-vector-to-gpu", "func::FuncOp"> {
- let summary = "Pass to convert vector to gpu.";
- let constructor = "mlir::iree_compiler::createLLVMGPUVectorToGPU()";
-}
-
-def LLVMGPUTensorPad :
- Pass<"iree-llvmgpu-tensor-pad", "func::FuncOp"> {
- let summary = "Pass to pad out tensors up to static dimensions.";
- let constructor = "mlir::iree_compiler::createLLVMGPUTensorPadPass()";
-}
-
-//------------------------------------------------------------------------------
-// SPIR-V
-//------------------------------------------------------------------------------
-
-def ConvertToSPIRV : Pass<"iree-convert-to-spirv", "ModuleOp"> {
- let summary = "Perform the final conversion to SPIR-V dialect";
- let constructor = "mlir::iree_compiler::createConvertToSPIRVPass()";
- let options = [
- Option<"enableFastMathOption", "enable-fast-math", "bool", /*default=*/"false",
- "Enable fast math mode during type conversion (i.e. assume no NaN/infinity)">,
- Option<"indexBitsOption", "index-bits", "unsigned", /*default=*/"32",
- "Specify the bit widths for SPIR-V indices">,
- ];
-}
-
-def SPIRVVectorToGPUSubgroupMMA :
- Pass<"iree-spirv-vector-to-gpu-subgroup-mma-ops", "func::FuncOp"> {
- let summary = "Pass to convert vector ops to GPU subgroup MMA ops.";
- let constructor = "mlir::iree_compiler::createSPIRVVectorToGPUSubgroupMMAOpsPass()";
-}
-
-def SPIRVLowerExecutableTarget :
- Pass<"iree-spirv-lower-executable-target-pass",
- "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
- let summary = "Lower the executable target to SPIR-V using one of the "
- "IREE::HAL::DispatchLoweringPassPipeline";
- let constructor =
- "mlir::iree_compiler::createSPIRVLowerExecutableTargetPass()";
-}
-
-def SPIRVTile : Pass<"iree-spirv-tile", "func::FuncOp"> {
- let summary = "Tile Linalg ops with tensor semantics to invocations";
- let constructor = "mlir::iree_compiler::createSPIRVTilePass()";
-}
-
-def SPIRVDistribute : Pass<"iree-spirv-distribute", "func::FuncOp"> {
- let summary = "Distribute tiled loop nests to invocations";
- let constructor = "mlir::iree_compiler::createSPIRVDistributePass()";
-}
-
-def SPIRVTileAndDistribute : Pass<"iree-spirv-tile-and-distribute", "func::FuncOp"> {
- let summary = "Tile and distribute Linalg ops with buffer semantics to "
- "invocations";
- let constructor = "mlir::iree_compiler::createSPIRVTileAndDistributePass()";
-}
-
-def SPIRVTileToCooperativeOps : Pass<
- "iree-spirv-tile-to-cooperative-ops", "func::FuncOp"> {
- let summary = "Tile Linalg ops with buffer semantics to subgroups and "
- "vectorize to vector ops suitable for lowering to SPIR-V "
- "cooperative ops";
- let constructor =
- "mlir::iree_compiler::createSPIRVTileToCooperativeOpsPass()";
-}
-
-def SPIRVVectorizeToCooperativeOps : Pass<
- "iree-spirv-vectorize-to-cooperative-ops", "func::FuncOp"> {
- let summary = "Tile Linalg ops with buffer semantics to subgroups and "
- "vectorize to vector ops suitable for lowering to SPIR-V "
- "cooperative ops";
- let constructor =
- "mlir::iree_compiler::createSPIRVVectorizeToCooperativeOpsPass()";
-}
-
-def SPIRVTileAndPromote : Pass<"iree-spirv-tile-and-promote", "func::FuncOp"> {
- let summary = "Promote tiled Linalg ops with buffer semantics to use "
- "workgroup memory and then tile to invocations";
- let constructor =
- "mlir::iree_compiler::createSPIRVTileAndPromotePass()";
- let options = [
- Option<"promoteC", "promote-c", "bool", /*default=*/"false",
- "Promote C matrix to use shared memory">,
- Option<"skipThread", "skip-thread", "bool", /*default=*/"false",
- "Skip tiling and distributing to GPU threads">,
- ];
-}
-
-def SPIRVVectorize : Pass<"iree-spirv-vectorize", "func::FuncOp"> {
- let summary = "Vectorize Linalg ops with buffer semantics";
- let constructor = "mlir::iree_compiler::createSPIRVVectorizePass()";
-}
-
-def SPIRVVectorizeLoadStore :
- Pass<"iree-spirv-vectorize-load-store", "ModuleOp"> {
- let summary = "Vectorize load/store of memrefs for better memory access";
- let constructor = "mlir::iree_compiler::createSPIRVVectorizeLoadStore()";
-}
-
-def SPIRVBreakDownLargeVector : Pass<"iree-spirv-breakdown-large-vector",
- "func::FuncOp"> {
- let summary = "Break down vectors not natively supported by SPIR-V";
- let constructor = "mlir::iree_compiler::createSPIRVBreakDownLargeVectorPass()";
-}
-
-def SPIRVCreateFastSlowPath :
- Pass<"iree-spirv-create-fast-slow-path", "func::FuncOp"> {
- let summary = "Create separate fast and slow paths to handle padding";
- let constructor = "mlir::iree_compiler::createSPIRVCreateFastSlowPathPass()";
-}
-
-def SPIRVEmulateI64 :
- Pass<"iree-spirv-emulate-i64", "ModuleOp"> {
- let summary = "Emulate 64-bit integer ops with 32-bit integer ops";
- let constructor = "mlir::iree_compiler::createSPIRVEmulateI64Pass()";
-}
-
-def SPIRVEraseStorageBufferStaticShape :
- Pass<"iree-spirv-erase-storage-buffer-static-shape", "func::FuncOp"> {
- let summary = "Turn static shaped storage buffer subspan ops into dynamic shaped ones";
- let constructor = "mlir::iree_compiler::createSPIRVEraseStorageBufferStaticShapePass()";
-}
-
-def SPIRVMapMemRefStorageClass :
- Pass<"iree-spirv-map-memref-storage-class", "func::FuncOp"> {
- let summary = "Map MemRef memory spaces to SPIR-V storage classes";
- let constructor = "mlir::iree_compiler::createSPIRVMapMemRefStorageClassPass()";
-}
-
-def SPIRVAnnotateWinogradLoops : Pass<"iree-spirv-annotate-winograd-loops", "func::FuncOp"> {
- let summary = "Annotate innermost Winograd loops with spirv distribute attribute";
- let constructor = "mlir::iree_compiler::createSPIRVAnnotateWinogradLoopsPass()";
-}
-
-//------------------------------------------------------------------------------
-// VMVX Passes
-//------------------------------------------------------------------------------
-
-def VMVXMaterializeEncoding :
- Pass<"iree-vmvx-materialize-encoding", "func::FuncOp"> {
- let summary = "Materialize the encoding for tensor as specified by the backend";
- let constructor = "mlir::iree_compiler::createVMVXMaterializeEncodingPass()";
-}
-
-
-def VMVXLowerLinalgMicrokernels :
- Pass<"iree-vmvx-lower-linalg-microkernels", ""> {
- let summary =
- "Lowers linalg ops to the VMVX microkernel library";
- let constructor = "mlir::iree_compiler::createVMVXLowerLinalgMicrokernelsPass()";
- let options = [
- Option<"warnOnUnconverted", "warn-on-unconverted", "bool",
- /*default=*/"false",
- "Warns on any unconverted linalg ops which remain live">
- ];
-}
-
-def VMVXLinkExecutables :
- Pass<"iree-vmvx-link-executables", "mlir::ModuleOp"> {
- let summary = "Links VMVX HAL executables within the top-level program module.";
- let constructor = "mlir::iree_compiler::createVMVXLinkExecutablesPass()";
-}
-
-def VMVXAssignConstantOrdinals :
- Pass<"iree-vmvx-assign-constant-ordinals", "IREE::HAL::ExecutableVariantOp"> {
- let summary = "Assigns executable constant ordinals across all VMVX variants.";
- let constructor = "mlir::iree_compiler::createVMVXAssignConstantOrdinalsPass()";
-}
-
-//------------------------------------------------------------------------------
-// WGSL Passes
-//------------------------------------------------------------------------------
-
-def WGSLReplacePushConstants :
- Pass<"iree-wgsl-replace-push-constants", "func::FuncOp"> {
- let summary =
- "Replaces push constant loads with binding loads for when using "
- "WGSL without push constant support";
- let constructor = "mlir::iree_compiler::createWGSLReplacePushConstantsPass()";
-}
-
-//------------------------------------------------------------------------------
-// Test Passes
-//------------------------------------------------------------------------------
-
-def TestLLVMGPUScalarizeMathOp :
- Pass<"iree-test-llvmgpu-legalize-ops", "ModuleOp"> {
- let summary = "Test pass for several legalization patterns.";
- let constructor = "mlir::iree_compiler::createTestLLVMGPULegalizePass()";
-}
-
-#endif // IREE_DIALECT_FLOW_PASSES
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel b/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel
index 5acc07f..7876187 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/BUILD.bazel
@@ -4,7 +4,7 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library")
+load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library", "iree_gentbl_cc_library")
package(
default_visibility = ["//visibility:public"],
@@ -12,6 +12,39 @@
licenses = ["notice"], # Apache 2.0
)
+iree_gentbl_cc_library(
+ name = "PassesIncGen",
+ tbl_outs = [
+ (
+ ["--gen-pass-decls"],
+ "Passes.h.inc",
+ ),
+ ],
+ tblgen = "@llvm-project//mlir:mlir-tblgen",
+ td_file = "Passes.td",
+ deps = ["@llvm-project//mlir:PassBaseTdFiles"],
+)
+
+iree_compiler_cc_library(
+ name = "PassHeaders",
+ hdrs = [
+ "PassDetail.h",
+ "Passes.h",
+ "Passes.h.inc",
+ ],
+ deps = [
+ ":PassesIncGen",
+ "//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
+ "//compiler/src/iree/compiler/Dialect/HAL/IR",
+ "//compiler/src/iree/compiler/Utils",
+ "@llvm-project//mlir:LinalgTransforms",
+ "@llvm-project//mlir:MemRefDialect",
+ "@llvm-project//mlir:NVGPUDialect",
+ "@llvm-project//mlir:Pass",
+ "@llvm-project//mlir:Transforms",
+ ],
+)
+
iree_compiler_cc_library(
name = "SPIRV",
srcs = [
@@ -43,13 +76,14 @@
],
hdrs = [
"KernelConfig.h",
- "SPIRVPasses.h",
+ "Passes.h",
"Utils.h",
],
deps = [
- "//compiler/src/iree/compiler/Codegen:PassHeaders",
+ ":PassHeaders",
+ ":PassesIncGen",
"//compiler/src/iree/compiler/Codegen/Common",
- "//compiler/src/iree/compiler/Codegen/Common:CommonPasses",
+ "//compiler/src/iree/compiler/Codegen/Common:TransformDialectInterpreterPass",
"//compiler/src/iree/compiler/Codegen/Common/GPU:CommonGPUPasses",
"//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
"//compiler/src/iree/compiler/Codegen/Transforms",
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt
index df2b7af..71ff664 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/CMakeLists.txt
@@ -10,12 +10,41 @@
iree_add_all_subdirs()
+iree_tablegen_library(
+ NAME
+ PassesIncGen
+ TD_FILE
+ "Passes.td"
+ OUTS
+ --gen-pass-decls Passes.h.inc
+)
+
+iree_cc_library(
+ NAME
+ PassHeaders
+ HDRS
+ "PassDetail.h"
+ "Passes.h"
+ "Passes.h.inc"
+ DEPS
+ ::PassesIncGen
+ MLIRLinalgTransforms
+ MLIRMemRefDialect
+ MLIRNVGPUDialect
+ MLIRPass
+ MLIRTransforms
+ iree::compiler::Codegen::Dialect::IREECodegenDialect
+ iree::compiler::Dialect::HAL::IR
+ iree::compiler::Utils
+ PUBLIC
+)
+
iree_cc_library(
NAME
SPIRV
HDRS
"KernelConfig.h"
- "SPIRVPasses.h"
+ "Passes.h"
"Utils.h"
SRCS
"AMDConfig.cpp"
@@ -44,6 +73,8 @@
"Utils.cpp"
"Verifiers.cpp"
DEPS
+ ::PassHeaders
+ ::PassesIncGen
IREELinalgExtDialect
IREELinalgExtPasses
IREELinalgExtTransforms
@@ -102,10 +133,9 @@
MLIRVectorToSPIRV
MLIRVectorTransforms
iree::compiler::Codegen::Common
- iree::compiler::Codegen::Common::CommonPasses
iree::compiler::Codegen::Common::GPU::CommonGPUPasses
+ iree::compiler::Codegen::Common::TransformDialectInterpreterPass
iree::compiler::Codegen::Dialect::IREECodegenDialect
- iree::compiler::Codegen::PassHeaders
iree::compiler::Codegen::Transforms
iree::compiler::Codegen::Utils
iree::compiler::Dialect::Flow::IR
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/ConvertToSPIRVPass.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/ConvertToSPIRVPass.cpp
index e9814ca..8cd2d21 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/ConvertToSPIRVPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/ConvertToSPIRVPass.cpp
@@ -16,8 +16,8 @@
#include <tuple>
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/PassDetail.h b/compiler/src/iree/compiler/Codegen/SPIRV/PassDetail.h
similarity index 70%
rename from compiler/src/iree/compiler/Codegen/PassDetail.h
rename to compiler/src/iree/compiler/Codegen/SPIRV/PassDetail.h
index fea75f0..d41d492 100644
--- a/compiler/src/iree/compiler/Codegen/PassDetail.h
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/PassDetail.h
@@ -1,11 +1,11 @@
-// Copyright 2021 The IREE Authors
+// Copyright 2023 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#ifndef IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
-#define IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#ifndef IREE_COMPILER_CODEGEN_SPIRV_PASS_DETAIL_H_
+#define IREE_COMPILER_CODEGEN_SPIRV_PASS_DETAIL_H_
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
@@ -18,9 +18,9 @@
namespace iree_compiler {
#define GEN_PASS_CLASSES
-#include "iree/compiler/Codegen/Passes.h.inc"
+#include "iree/compiler/Codegen/SPIRV/Passes.h.inc"
} // namespace iree_compiler
} // namespace mlir
-#endif // IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#endif // IREE_COMPILER_CODEGEN_SPIRV_PASS_DETAIL_H_
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp
index 6b62b94..93b0b1c 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.cpp
@@ -14,11 +14,11 @@
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
#include "iree-dialects/Dialect/LinalgTransform/Passes.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/Common/GPU/CommonGPUPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/GPU/Passes.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/SPIRV/KernelConfig.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
@@ -603,5 +603,26 @@
});
}
+//===---------------------------------------------------------------------===//
+// Register SPIRV Passes
+//===---------------------------------------------------------------------===//
+
+namespace {
+#define GEN_PASS_REGISTRATION
+#include "iree/compiler/Codegen/SPIRV/Passes.h.inc"
+} // namespace
+
+void registerCodegenSPIRVPasses() {
+ // Generated.
+ registerPasses();
+
+ static PassPipelineRegistration<> LinalgSPIRVPipeline(
+ "iree-codegen-linalg-to-spirv-pipeline",
+ "Runs the progressive lowering pipeline from linalg to SPIR-V",
+ [](OpPassManager &passManager) {
+ buildSPIRVCodegenPassPipeline(passManager, /*enableFastMath=*/false);
+ });
+}
+
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVPasses.h b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h
similarity index 96%
rename from compiler/src/iree/compiler/Codegen/SPIRV/SPIRVPasses.h
rename to compiler/src/iree/compiler/Codegen/SPIRV/Passes.h
index 1eb7e61..1eb2af7 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVPasses.h
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.h
@@ -160,6 +160,12 @@
IREE::Codegen::TranslationInfoAttr translationInfo,
ArrayRef<int64_t> workgroupSize);
+//----------------------------------------------------------------------------//
+// Register SPIRV Passes
+//----------------------------------------------------------------------------//
+
+void registerCodegenSPIRVPasses();
+
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td
new file mode 100644
index 0000000..fe003c3
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/Passes.td
@@ -0,0 +1,135 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef IREE_CODEGEN_SPIRV_PASSES
+#define IREE_CODEGEN_SPIRV_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+//===---------------------------------------------------------------------===//
+// SPIRV passes (keep alphabetical)
+//===---------------------------------------------------------------------===//
+
+def ConvertToSPIRV : Pass<"iree-convert-to-spirv", "ModuleOp"> {
+ let summary = "Perform the final conversion to SPIR-V dialect";
+ let constructor = "mlir::iree_compiler::createConvertToSPIRVPass()";
+ let options = [
+ Option<"enableFastMathOption", "enable-fast-math", "bool", /*default=*/"false",
+ "Enable fast math mode during type conversion (i.e. assume no NaN/infinity)">,
+ Option<"indexBitsOption", "index-bits", "unsigned", /*default=*/"32",
+ "Specify the bit widths for SPIR-V indices">,
+ ];
+}
+
+def SPIRVAnnotateWinogradLoops : Pass<"iree-spirv-annotate-winograd-loops", "func::FuncOp"> {
+ let summary = "Annotate innermost Winograd loops with spirv distribute attribute";
+ let constructor = "mlir::iree_compiler::createSPIRVAnnotateWinogradLoopsPass()";
+}
+
+def SPIRVBreakDownLargeVector : Pass<"iree-spirv-breakdown-large-vector",
+ "func::FuncOp"> {
+ let summary = "Break down vectors not natively supported by SPIR-V";
+ let constructor = "mlir::iree_compiler::createSPIRVBreakDownLargeVectorPass()";
+}
+
+def SPIRVCreateFastSlowPath :
+ Pass<"iree-spirv-create-fast-slow-path", "func::FuncOp"> {
+ let summary = "Create separate fast and slow paths to handle padding";
+ let constructor = "mlir::iree_compiler::createSPIRVCreateFastSlowPathPass()";
+}
+
+def SPIRVDistribute : Pass<"iree-spirv-distribute", "func::FuncOp"> {
+ let summary = "Distribute tiled loop nests to invocations";
+ let constructor = "mlir::iree_compiler::createSPIRVDistributePass()";
+}
+
+def SPIRVEmulateI64 :
+ Pass<"iree-spirv-emulate-i64", "ModuleOp"> {
+ let summary = "Emulate 64-bit integer ops with 32-bit integer ops";
+ let constructor = "mlir::iree_compiler::createSPIRVEmulateI64Pass()";
+}
+
+def SPIRVEraseStorageBufferStaticShape :
+ Pass<"iree-spirv-erase-storage-buffer-static-shape", "func::FuncOp"> {
+ let summary = "Turn static shaped storage buffer subspan ops into dynamic shaped ones";
+ let constructor = "mlir::iree_compiler::createSPIRVEraseStorageBufferStaticShapePass()";
+}
+
+def SPIRVLowerExecutableTarget :
+ Pass<"iree-spirv-lower-executable-target-pass",
+ "mlir::iree_compiler::IREE::HAL::ExecutableVariantOp"> {
+ let summary = "Lower the executable target to SPIR-V using one of the "
+ "IREE::HAL::DispatchLoweringPassPipeline";
+ let constructor =
+ "mlir::iree_compiler::createSPIRVLowerExecutableTargetPass()";
+}
+
+def SPIRVMapMemRefStorageClass :
+ Pass<"iree-spirv-map-memref-storage-class", "func::FuncOp"> {
+ let summary = "Map MemRef memory spaces to SPIR-V storage classes";
+ let constructor = "mlir::iree_compiler::createSPIRVMapMemRefStorageClassPass()";
+}
+
+def SPIRVTile : Pass<"iree-spirv-tile", "func::FuncOp"> {
+ let summary = "Tile Linalg ops with tensor semantics to invocations";
+ let constructor = "mlir::iree_compiler::createSPIRVTilePass()";
+}
+
+def SPIRVTileAndDistribute : Pass<"iree-spirv-tile-and-distribute", "func::FuncOp"> {
+ let summary = "Tile and distribute Linalg ops with buffer semantics to "
+ "invocations";
+ let constructor = "mlir::iree_compiler::createSPIRVTileAndDistributePass()";
+}
+
+def SPIRVTileAndPromote : Pass<"iree-spirv-tile-and-promote", "func::FuncOp"> {
+ let summary = "Promote tiled Linalg ops with buffer semantics to use "
+ "workgroup memory and then tile to invocations";
+ let constructor =
+ "mlir::iree_compiler::createSPIRVTileAndPromotePass()";
+ let options = [
+ Option<"promoteC", "promote-c", "bool", /*default=*/"false",
+ "Promote C matrix to use shared memory">,
+ Option<"skipThread", "skip-thread", "bool", /*default=*/"false",
+ "Skip tiling and distributing to GPU threads">,
+ ];
+}
+
+def SPIRVTileToCooperativeOps : Pass<
+ "iree-spirv-tile-to-cooperative-ops", "func::FuncOp"> {
+ let summary = "Tile Linalg ops with buffer semantics to subgroups and "
+ "vectorize to vector ops suitable for lowering to SPIR-V "
+ "cooperative ops";
+ let constructor =
+ "mlir::iree_compiler::createSPIRVTileToCooperativeOpsPass()";
+}
+
+def SPIRVVectorize : Pass<"iree-spirv-vectorize", "func::FuncOp"> {
+ let summary = "Vectorize Linalg ops with buffer semantics";
+ let constructor = "mlir::iree_compiler::createSPIRVVectorizePass()";
+}
+
+def SPIRVVectorizeLoadStore :
+ Pass<"iree-spirv-vectorize-load-store", "ModuleOp"> {
+ let summary = "Vectorize load/store of memrefs for better memory access";
+ let constructor = "mlir::iree_compiler::createSPIRVVectorizeLoadStore()";
+}
+
+def SPIRVVectorizeToCooperativeOps : Pass<
+ "iree-spirv-vectorize-to-cooperative-ops", "func::FuncOp"> {
+ let summary = "Tile Linalg ops with buffer semantics to subgroups and "
+ "vectorize to vector ops suitable for lowering to SPIR-V "
+ "cooperative ops";
+ let constructor =
+ "mlir::iree_compiler::createSPIRVVectorizeToCooperativeOpsPass()";
+}
+
+def SPIRVVectorToGPUSubgroupMMA :
+ Pass<"iree-spirv-vector-to-gpu-subgroup-mma-ops", "func::FuncOp"> {
+ let summary = "Pass to convert vector ops to GPU subgroup MMA ops.";
+ let constructor = "mlir::iree_compiler::createSPIRVVectorToGPUSubgroupMMAOpsPass()";
+}
+
+#endif // IREE_CODEGEN_SPIRV_PASSES
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVAnnotateWinogradLoops.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVAnnotateWinogradLoops.cpp
index f779b6f..0bc3c8b 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVAnnotateWinogradLoops.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVAnnotateWinogradLoops.cpp
@@ -4,7 +4,7 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVBreakDownLargeVector.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVBreakDownLargeVector.cpp
index 35e9b05..b040edd 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVBreakDownLargeVector.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVBreakDownLargeVector.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVCreateFastSlowPath.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVCreateFastSlowPath.cpp
index ce1514b..3ab9fd3 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVCreateFastSlowPath.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVCreateFastSlowPath.cpp
@@ -13,8 +13,8 @@
//
//===----------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/Support/Debug.h"
#include "mlir/Analysis/SliceAnalysis.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVDistribute.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVDistribute.cpp
index 1866f83..c01ac1b 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVDistribute.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVDistribute.cpp
@@ -11,8 +11,8 @@
//
//===----------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEmulateI64.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEmulateI64.cpp
index 7a96fb8..2bfd76c 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEmulateI64.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEmulateI64.cpp
@@ -11,8 +11,8 @@
//
//===----------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEraseStorageBufferStaticShape.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEraseStorageBufferStaticShape.cpp
index 7eb3073..457444d 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEraseStorageBufferStaticShape.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVEraseStorageBufferStaticShape.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/HAL/IR/HALTypes.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp
index 71350d2..6852f7a 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVLowerExecutableTargetPass.cpp
@@ -7,9 +7,9 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/SPIRV/KernelConfig.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "llvm/Support/Debug.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMapMemRefStorageClass.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMapMemRefStorageClass.cpp
index 226cdba..25d55c7 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMapMemRefStorageClass.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVMapMemRefStorageClass.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Dialect/HAL/IR/HALTypes.h"
#include "mlir/Conversion/MemRefToSPIRV/MemRefToSPIRV.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTile.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTile.cpp
index c5bb074..525399b 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTile.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTile.cpp
@@ -12,10 +12,10 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtOps.h"
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndDistribute.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndDistribute.cpp
index e84b16b..5b90bf9 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndDistribute.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndDistribute.cpp
@@ -14,10 +14,10 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtOps.h"
#include "iree-dialects/Dialect/LinalgExt/Passes/Transforms.h"
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndPromote.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndPromote.cpp
index bdd208f..076fed3 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndPromote.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndPromote.cpp
@@ -12,11 +12,11 @@
//===----------------------------------------------------------------------===//
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
#include "iree/compiler/Codegen/Common/GPU/GPUPatterns.h"
-#include "iree/compiler/Codegen/PassDetail.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/SPIRV/KernelConfig.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndVectorizeToCooperativeOps.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndVectorizeToCooperativeOps.cpp
index 27ced4d..4f50c4c 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndVectorizeToCooperativeOps.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVTileAndVectorizeToCooperativeOps.cpp
@@ -15,12 +15,12 @@
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
#include "iree/compiler/Codegen/Common/GPU/GPUPatterns.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
#include "iree/compiler/Codegen/Dialect/LoweringConfig.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/SPIRV/KernelConfig.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Codegen/Utils/GPUUtils.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorToGPUSubgroupMMAOps.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorToGPUSubgroupMMAOps.cpp
index ab48475..43d48a6 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorToGPUSubgroupMMAOps.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorToGPUSubgroupMMAOps.cpp
@@ -5,8 +5,8 @@
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "iree/compiler/Codegen/Common/GPU/GPUPatterns.h"
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "mlir/Conversion/VectorToGPU/VectorToGPU.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorize.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorize.cpp
index 6fe6800..93a8e1d 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorize.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorize.cpp
@@ -12,9 +12,9 @@
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
#include "iree-dialects/Dialect/LinalgExt/Transforms/Transforms.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Codegen/Transforms/Transforms.h"
#include "iree/compiler/Codegen/Utils/MarkerUtils.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorizeLoadStore.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorizeLoadStore.cpp
index 1989bd9..3e3674d 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorizeLoadStore.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/SPIRVVectorizeLoadStore.cpp
@@ -12,8 +12,8 @@
//
//===----------------------------------------------------------------------===//
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/Util/IR/UtilOps.h"
#include "llvm/ADT/TypeSwitch.h"
diff --git a/compiler/src/iree/compiler/Codegen/SPIRV/Verifiers.cpp b/compiler/src/iree/compiler/Codegen/SPIRV/Verifiers.cpp
index 83aceab..1191c65 100644
--- a/compiler/src/iree/compiler/Codegen/SPIRV/Verifiers.cpp
+++ b/compiler/src/iree/compiler/Codegen/SPIRV/Verifiers.cpp
@@ -4,9 +4,9 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/SPIRV/KernelConfig.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/PassDetail.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Codegen/SPIRV/Utils.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
#include "llvm/Support/Debug.h"
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/BUILD.bazel b/compiler/src/iree/compiler/Codegen/VMVX/BUILD.bazel
index fcc30b9..763c922 100644
--- a/compiler/src/iree/compiler/Codegen/VMVX/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/VMVX/BUILD.bazel
@@ -4,7 +4,7 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library")
+load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library", "iree_gentbl_cc_library")
package(
default_visibility = ["//visibility:public"],
@@ -12,6 +12,38 @@
licenses = ["notice"], # Apache 2.0
)
+iree_gentbl_cc_library(
+ name = "PassesIncGen",
+ tbl_outs = [
+ (
+ ["--gen-pass-decls"],
+ "Passes.h.inc",
+ ),
+ ],
+ tblgen = "@llvm-project//mlir:mlir-tblgen",
+ td_file = "Passes.td",
+ deps = ["@llvm-project//mlir:PassBaseTdFiles"],
+)
+
+iree_compiler_cc_library(
+ name = "PassHeaders",
+ hdrs = [
+ "PassDetail.h",
+ "Passes.h",
+ "Passes.h.inc",
+ ],
+ deps = [
+ ":PassesIncGen",
+ "//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
+ "//compiler/src/iree/compiler/Dialect/HAL/IR",
+ "//compiler/src/iree/compiler/Utils",
+ "@llvm-project//mlir:LinalgTransforms",
+ "@llvm-project//mlir:MemRefDialect",
+ "@llvm-project//mlir:Pass",
+ "@llvm-project//mlir:Transforms",
+ ],
+)
+
iree_compiler_cc_library(
name = "VMVX",
srcs = [
@@ -23,12 +55,12 @@
],
hdrs = [
"EncodingInfo.h",
- "VMVXPasses.h",
+ "Passes.h",
],
deps = [
- "//compiler/src/iree/compiler/Codegen:PassHeaders",
+ ":PassHeaders",
+ ":PassesIncGen",
"//compiler/src/iree/compiler/Codegen/Common",
- "//compiler/src/iree/compiler/Codegen/Common:CommonPasses",
"//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
"//compiler/src/iree/compiler/Codegen/Utils",
"//compiler/src/iree/compiler/Dialect/Flow/IR",
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/VMVX/CMakeLists.txt
index d9dea0d..3d1b599 100644
--- a/compiler/src/iree/compiler/Codegen/VMVX/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/VMVX/CMakeLists.txt
@@ -10,12 +10,40 @@
iree_add_all_subdirs()
+iree_tablegen_library(
+ NAME
+ PassesIncGen
+ TD_FILE
+ "Passes.td"
+ OUTS
+ --gen-pass-decls Passes.h.inc
+)
+
+iree_cc_library(
+ NAME
+ PassHeaders
+ HDRS
+ "PassDetail.h"
+ "Passes.h"
+ "Passes.h.inc"
+ DEPS
+ ::PassesIncGen
+ MLIRLinalgTransforms
+ MLIRMemRefDialect
+ MLIRPass
+ MLIRTransforms
+ iree::compiler::Codegen::Dialect::IREECodegenDialect
+ iree::compiler::Dialect::HAL::IR
+ iree::compiler::Utils
+ PUBLIC
+)
+
iree_cc_library(
NAME
VMVX
HDRS
"EncodingInfo.h"
- "VMVXPasses.h"
+ "Passes.h"
SRCS
"LowerLinalgMicrokernels.cpp"
"Passes.cpp"
@@ -23,6 +51,8 @@
"VMVXLinkExecutables.cpp"
"VMVXMaterializeEncodingPass.cpp"
DEPS
+ ::PassHeaders
+ ::PassesIncGen
IREELinalgExtDialect
IREELinalgExtPasses
LLVMSupport
@@ -40,9 +70,7 @@
MLIRTransforms
iree::builtins::ukernel::exported_bits
iree::compiler::Codegen::Common
- iree::compiler::Codegen::Common::CommonPasses
iree::compiler::Codegen::Dialect::IREECodegenDialect
- iree::compiler::Codegen::PassHeaders
iree::compiler::Codegen::Utils
iree::compiler::Dialect::Flow::IR
iree::compiler::Dialect::HAL::IR
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/LowerLinalgMicrokernels.cpp b/compiler/src/iree/compiler/Codegen/VMVX/LowerLinalgMicrokernels.cpp
index 2be06ad..a9a31b6 100644
--- a/compiler/src/iree/compiler/Codegen/VMVX/LowerLinalgMicrokernels.cpp
+++ b/compiler/src/iree/compiler/Codegen/VMVX/LowerLinalgMicrokernels.cpp
@@ -6,8 +6,8 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtOps.h"
#include "iree/builtins/ukernel/exported_bits.h"
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/VMVX/VMVXPasses.h"
+#include "iree/compiler/Codegen/VMVX/PassDetail.h"
+#include "iree/compiler/Codegen/VMVX/Passes.h"
#include "iree/compiler/Dialect/Util/IR/UtilDialect.h"
#include "iree/compiler/Dialect/VMVX/IR/VMVXDialect.h"
#include "iree/compiler/Dialect/VMVX/IR/VMVXOps.h"
diff --git a/compiler/src/iree/compiler/Codegen/PassDetail.h b/compiler/src/iree/compiler/Codegen/VMVX/PassDetail.h
similarity index 65%
copy from compiler/src/iree/compiler/Codegen/PassDetail.h
copy to compiler/src/iree/compiler/Codegen/VMVX/PassDetail.h
index fea75f0..c698ea2 100644
--- a/compiler/src/iree/compiler/Codegen/PassDetail.h
+++ b/compiler/src/iree/compiler/Codegen/VMVX/PassDetail.h
@@ -1,26 +1,25 @@
-// Copyright 2021 The IREE Authors
+// Copyright 2023 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#ifndef IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
-#define IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#ifndef IREE_COMPILER_CODEGEN_VMVX_PASS_DETAIL_H_
+#define IREE_COMPILER_CODEGEN_VMVX_PASS_DETAIL_H_
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
#include "mlir/Pass/Pass.h"
namespace mlir {
namespace iree_compiler {
#define GEN_PASS_CLASSES
-#include "iree/compiler/Codegen/Passes.h.inc"
+#include "iree/compiler/Codegen/VMVX/Passes.h.inc"
} // namespace iree_compiler
} // namespace mlir
-#endif // IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#endif // IREE_COMPILER_CODEGEN_VMVX_PASS_DETAIL_H_
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/Passes.cpp b/compiler/src/iree/compiler/Codegen/VMVX/Passes.cpp
index 0abfedb..075a063 100644
--- a/compiler/src/iree/compiler/Codegen/VMVX/Passes.cpp
+++ b/compiler/src/iree/compiler/Codegen/VMVX/Passes.cpp
@@ -6,8 +6,8 @@
#include "mlir/Transforms/Passes.h"
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/VMVX/VMVXPasses.h"
+#include "iree/compiler/Codegen/VMVX/PassDetail.h"
+#include "iree/compiler/Codegen/VMVX/Passes.h"
#include "mlir/Pass/PassManager.h"
namespace mlir {
@@ -29,5 +29,26 @@
createVMVXAssignConstantOrdinalsPass());
}
+//===---------------------------------------------------------------------===//
+// Register VMVX Passes
+//===---------------------------------------------------------------------===//
+
+namespace {
+#define GEN_PASS_REGISTRATION
+#include "iree/compiler/Codegen/VMVX/Passes.h.inc"
+} // namespace
+
+void registerCodegenVMVXPasses() {
+ // Generated.
+ registerPasses();
+
+ static PassPipelineRegistration<> VMVXLinkingPipeline(
+ "iree-codegen-vmvx-linking-pipeline",
+ "Runs the VMVX HAL executable linking pipeline",
+ [](OpPassManager &passManager) {
+ buildVMVXLinkingPassPipeline(passManager);
+ });
+}
+
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/VMVXPasses.h b/compiler/src/iree/compiler/Codegen/VMVX/Passes.h
similarity index 89%
rename from compiler/src/iree/compiler/Codegen/VMVX/VMVXPasses.h
rename to compiler/src/iree/compiler/Codegen/VMVX/Passes.h
index ac3a5f5..fbcee95 100644
--- a/compiler/src/iree/compiler/Codegen/VMVX/VMVXPasses.h
+++ b/compiler/src/iree/compiler/Codegen/VMVX/Passes.h
@@ -18,6 +18,7 @@
namespace mlir {
namespace iree_compiler {
+
//------------------------------------------------------------------------------
// VMVX passes
//------------------------------------------------------------------------------
@@ -44,6 +45,13 @@
/// Populates passes needed to link HAL executables across VMVX targets.
void buildVMVXLinkingPassPipeline(OpPassManager &passManager);
+
+//----------------------------------------------------------------------------//
+// Register VMVX Passes
+//----------------------------------------------------------------------------//
+
+void registerCodegenVMVXPasses();
+
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/Passes.td b/compiler/src/iree/compiler/Codegen/VMVX/Passes.td
new file mode 100644
index 0000000..92ea2fd
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/VMVX/Passes.td
@@ -0,0 +1,46 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef IREE_CODEGEN_VMVX_PASSES
+#define IREE_CODEGEN_VMVX_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+//===---------------------------------------------------------------------===//
+// VMVX passes (keep alphabetical)
+//===---------------------------------------------------------------------===//
+
+def VMVXAssignConstantOrdinals :
+ Pass<"iree-vmvx-assign-constant-ordinals", "IREE::HAL::ExecutableVariantOp"> {
+ let summary = "Assigns executable constant ordinals across all VMVX variants.";
+ let constructor = "mlir::iree_compiler::createVMVXAssignConstantOrdinalsPass()";
+}
+
+def VMVXLinkExecutables :
+ Pass<"iree-vmvx-link-executables", "mlir::ModuleOp"> {
+ let summary = "Links VMVX HAL executables within the top-level program module.";
+ let constructor = "mlir::iree_compiler::createVMVXLinkExecutablesPass()";
+}
+
+def VMVXLowerLinalgMicrokernels :
+ Pass<"iree-vmvx-lower-linalg-microkernels", ""> {
+ let summary =
+ "Lowers linalg ops to the VMVX microkernel library";
+ let constructor = "mlir::iree_compiler::createVMVXLowerLinalgMicrokernelsPass()";
+ let options = [
+ Option<"warnOnUnconverted", "warn-on-unconverted", "bool",
+ /*default=*/"false",
+ "Warns on any unconverted linalg ops which remain live">
+ ];
+}
+
+def VMVXMaterializeEncoding :
+ Pass<"iree-vmvx-materialize-encoding", "func::FuncOp"> {
+ let summary = "Materialize the encoding for tensor as specified by the backend";
+ let constructor = "mlir::iree_compiler::createVMVXMaterializeEncodingPass()";
+}
+
+#endif // IREE_CODEGEN_VMVX_PASSES
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/VMVXAssignConstantOrdinals.cpp b/compiler/src/iree/compiler/Codegen/VMVX/VMVXAssignConstantOrdinals.cpp
index f3f6702..d370251 100644
--- a/compiler/src/iree/compiler/Codegen/VMVX/VMVXAssignConstantOrdinals.cpp
+++ b/compiler/src/iree/compiler/Codegen/VMVX/VMVXAssignConstantOrdinals.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/VMVX/VMVXPasses.h"
+#include "iree/compiler/Codegen/VMVX/PassDetail.h"
+#include "iree/compiler/Codegen/VMVX/Passes.h"
#include "iree/compiler/Dialect/VM/IR/VMOps.h"
#include "mlir/Pass/Pass.h"
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/VMVXLinkExecutables.cpp b/compiler/src/iree/compiler/Codegen/VMVX/VMVXLinkExecutables.cpp
index f4403a2..eb5b4e9 100644
--- a/compiler/src/iree/compiler/Codegen/VMVX/VMVXLinkExecutables.cpp
+++ b/compiler/src/iree/compiler/Codegen/VMVX/VMVXLinkExecutables.cpp
@@ -4,9 +4,9 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Utils/LinkingUtils.h"
-#include "iree/compiler/Codegen/VMVX/VMVXPasses.h"
+#include "iree/compiler/Codegen/VMVX/PassDetail.h"
+#include "iree/compiler/Codegen/VMVX/Passes.h"
#include "iree/compiler/Dialect/VM/IR/VMOps.h"
#include "iree/compiler/Utils/ModuleUtils.h"
#include "llvm/Support/FormatVariadic.h"
diff --git a/compiler/src/iree/compiler/Codegen/VMVX/VMVXMaterializeEncodingPass.cpp b/compiler/src/iree/compiler/Codegen/VMVX/VMVXMaterializeEncodingPass.cpp
index 0a1bd91..a98f24b 100644
--- a/compiler/src/iree/compiler/Codegen/VMVX/VMVXMaterializeEncodingPass.cpp
+++ b/compiler/src/iree/compiler/Codegen/VMVX/VMVXMaterializeEncodingPass.cpp
@@ -8,10 +8,10 @@
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
#include "iree/compiler/Codegen/Common/EncodingInfo.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
-#include "iree/compiler/Codegen/PassDetail.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Codegen/VMVX/EncodingInfo.h"
-#include "iree/compiler/Codegen/VMVX/VMVXPasses.h"
+#include "iree/compiler/Codegen/VMVX/PassDetail.h"
+#include "iree/compiler/Codegen/VMVX/Passes.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
#include "iree/compiler/Dialect/HAL/IR/HALTypes.h"
#include "iree/compiler/Dialect/VMVX/IR/VMVXDialect.h"
diff --git a/compiler/src/iree/compiler/Codegen/WGSL/BUILD.bazel b/compiler/src/iree/compiler/Codegen/WGSL/BUILD.bazel
index b289cb7..3ee9d71 100644
--- a/compiler/src/iree/compiler/Codegen/WGSL/BUILD.bazel
+++ b/compiler/src/iree/compiler/Codegen/WGSL/BUILD.bazel
@@ -4,7 +4,7 @@
# See https://llvm.org/LICENSE.txt for license information.
# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library")
+load("//build_tools/bazel:build_defs.oss.bzl", "iree_compiler_cc_library", "iree_gentbl_cc_library")
package(
default_visibility = ["//visibility:public"],
@@ -12,16 +12,50 @@
licenses = ["notice"], # Apache 2.0
)
+iree_gentbl_cc_library(
+ name = "PassesIncGen",
+ tbl_outs = [
+ (
+ ["--gen-pass-decls"],
+ "Passes.h.inc",
+ ),
+ ],
+ tblgen = "@llvm-project//mlir:mlir-tblgen",
+ td_file = "Passes.td",
+ deps = ["@llvm-project//mlir:PassBaseTdFiles"],
+)
+
+iree_compiler_cc_library(
+ name = "PassHeaders",
+ hdrs = [
+ "PassDetail.h",
+ "Passes.h",
+ "Passes.h.inc",
+ ],
+ deps = [
+ ":PassesIncGen",
+ "//compiler/src/iree/compiler/Codegen/Dialect:IREECodegenDialect",
+ "//compiler/src/iree/compiler/Dialect/HAL/IR",
+ "//compiler/src/iree/compiler/Utils",
+ "@llvm-project//mlir:LinalgTransforms",
+ "@llvm-project//mlir:MemRefDialect",
+ "@llvm-project//mlir:Pass",
+ "@llvm-project//mlir:Transforms",
+ ],
+)
+
iree_compiler_cc_library(
name = "WGSL",
srcs = [
+ "Passes.cpp",
"WGSLReplacePushConstants.cpp",
],
hdrs = [
- "WGSLPasses.h",
+ "Passes.h",
],
deps = [
- "//compiler/src/iree/compiler/Codegen:PassHeaders",
+ ":PassHeaders",
+ ":PassesIncGen",
"//compiler/src/iree/compiler/Dialect/Flow/IR",
"//compiler/src/iree/compiler/Dialect/HAL/IR",
"//compiler/src/iree/compiler/Dialect/HAL/IR:HALDialect",
@@ -30,6 +64,7 @@
"@llvm-project//mlir:IR",
"@llvm-project//mlir:Pass",
"@llvm-project//mlir:TensorDialect",
+ "@llvm-project//mlir:Transforms",
"@llvm-project//mlir:VectorDialect",
],
)
diff --git a/compiler/src/iree/compiler/Codegen/WGSL/CMakeLists.txt b/compiler/src/iree/compiler/Codegen/WGSL/CMakeLists.txt
index 7b3a025..011de3d 100644
--- a/compiler/src/iree/compiler/Codegen/WGSL/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Codegen/WGSL/CMakeLists.txt
@@ -10,21 +10,52 @@
iree_add_all_subdirs()
+iree_tablegen_library(
+ NAME
+ PassesIncGen
+ TD_FILE
+ "Passes.td"
+ OUTS
+ --gen-pass-decls Passes.h.inc
+)
+
+iree_cc_library(
+ NAME
+ PassHeaders
+ HDRS
+ "PassDetail.h"
+ "Passes.h"
+ "Passes.h.inc"
+ DEPS
+ ::PassesIncGen
+ MLIRLinalgTransforms
+ MLIRMemRefDialect
+ MLIRPass
+ MLIRTransforms
+ iree::compiler::Codegen::Dialect::IREECodegenDialect
+ iree::compiler::Dialect::HAL::IR
+ iree::compiler::Utils
+ PUBLIC
+)
+
iree_cc_library(
NAME
WGSL
HDRS
- "WGSLPasses.h"
+ "Passes.h"
SRCS
+ "Passes.cpp"
"WGSLReplacePushConstants.cpp"
DEPS
+ ::PassHeaders
+ ::PassesIncGen
MLIRArithDialect
MLIRFuncDialect
MLIRIR
MLIRPass
MLIRTensorDialect
+ MLIRTransforms
MLIRVectorDialect
- iree::compiler::Codegen::PassHeaders
iree::compiler::Dialect::Flow::IR
iree::compiler::Dialect::HAL::IR
iree::compiler::Dialect::HAL::IR::HALDialect
diff --git a/compiler/src/iree/compiler/Codegen/PassDetail.h b/compiler/src/iree/compiler/Codegen/WGSL/PassDetail.h
similarity index 65%
copy from compiler/src/iree/compiler/Codegen/PassDetail.h
copy to compiler/src/iree/compiler/Codegen/WGSL/PassDetail.h
index fea75f0..b036bff 100644
--- a/compiler/src/iree/compiler/Codegen/PassDetail.h
+++ b/compiler/src/iree/compiler/Codegen/WGSL/PassDetail.h
@@ -1,26 +1,25 @@
-// Copyright 2021 The IREE Authors
+// Copyright 2023 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#ifndef IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
-#define IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#ifndef IREE_COMPILER_CODEGEN_WGSL_PASS_DETAIL_H_
+#define IREE_COMPILER_CODEGEN_WGSL_PASS_DETAIL_H_
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
-#include "mlir/Dialect/NVGPU/IR/NVGPUDialect.h"
#include "mlir/Pass/Pass.h"
namespace mlir {
namespace iree_compiler {
#define GEN_PASS_CLASSES
-#include "iree/compiler/Codegen/Passes.h.inc"
+#include "iree/compiler/Codegen/WGSL/Passes.h.inc"
} // namespace iree_compiler
} // namespace mlir
-#endif // IREE_COMPILER_CONVERSIONS_PASS_DETAIL_H_
+#endif // IREE_COMPILER_CODEGEN_WGSL_PASS_DETAIL_H_
diff --git a/compiler/src/iree/compiler/Codegen/WGSL/Passes.cpp b/compiler/src/iree/compiler/Codegen/WGSL/Passes.cpp
new file mode 100644
index 0000000..5d8c0df
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/WGSL/Passes.cpp
@@ -0,0 +1,31 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "mlir/Transforms/Passes.h"
+
+#include "iree/compiler/Codegen/WGSL/PassDetail.h"
+#include "iree/compiler/Codegen/WGSL/Passes.h"
+#include "mlir/Pass/PassManager.h"
+
+namespace mlir {
+namespace iree_compiler {
+
+//===---------------------------------------------------------------------===//
+// Register WGSL Passes
+//===---------------------------------------------------------------------===//
+
+namespace {
+#define GEN_PASS_REGISTRATION
+#include "iree/compiler/Codegen/WGSL/Passes.h.inc"
+} // namespace
+
+void registerCodegenWGSLPasses() {
+ // Generated.
+ registerPasses();
+}
+
+} // namespace iree_compiler
+} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/WGSL/WGSLPasses.h b/compiler/src/iree/compiler/Codegen/WGSL/Passes.h
similarity index 81%
rename from compiler/src/iree/compiler/Codegen/WGSL/WGSLPasses.h
rename to compiler/src/iree/compiler/Codegen/WGSL/Passes.h
index c1511c4..72becb2 100644
--- a/compiler/src/iree/compiler/Codegen/WGSL/WGSLPasses.h
+++ b/compiler/src/iree/compiler/Codegen/WGSL/Passes.h
@@ -23,6 +23,12 @@
std::unique_ptr<OperationPass<func::FuncOp>>
createWGSLReplacePushConstantsPass();
+//----------------------------------------------------------------------------//
+// Register WGSL Passes
+//----------------------------------------------------------------------------//
+
+void registerCodegenWGSLPasses();
+
} // namespace iree_compiler
} // namespace mlir
diff --git a/compiler/src/iree/compiler/Codegen/WGSL/Passes.td b/compiler/src/iree/compiler/Codegen/WGSL/Passes.td
new file mode 100644
index 0000000..174fc32
--- /dev/null
+++ b/compiler/src/iree/compiler/Codegen/WGSL/Passes.td
@@ -0,0 +1,24 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef IREE_CODEGEN_WGSL_PASSES
+#define IREE_CODEGEN_WGSL_PASSES
+
+include "mlir/Pass/PassBase.td"
+
+//===---------------------------------------------------------------------===//
+// WGSL passes (keep alphabetical)
+//===---------------------------------------------------------------------===//
+
+def WGSLReplacePushConstants :
+ Pass<"iree-wgsl-replace-push-constants", "func::FuncOp"> {
+ let summary =
+ "Replaces push constant loads with binding loads for when using "
+ "WGSL without push constant support";
+ let constructor = "mlir::iree_compiler::createWGSLReplacePushConstantsPass()";
+}
+
+#endif // IREE_CODEGEN_WGSL_PASSES
diff --git a/compiler/src/iree/compiler/Codegen/WGSL/WGSLReplacePushConstants.cpp b/compiler/src/iree/compiler/Codegen/WGSL/WGSLReplacePushConstants.cpp
index 864d3c8..f88f0b3 100644
--- a/compiler/src/iree/compiler/Codegen/WGSL/WGSLReplacePushConstants.cpp
+++ b/compiler/src/iree/compiler/Codegen/WGSL/WGSLReplacePushConstants.cpp
@@ -4,8 +4,8 @@
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-#include "iree/compiler/Codegen/PassDetail.h"
-#include "iree/compiler/Codegen/WGSL/WGSLPasses.h"
+#include "iree/compiler/Codegen/WGSL/PassDetail.h"
+#include "iree/compiler/Codegen/WGSL/Passes.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
#include "iree/compiler/Dialect/HAL/IR/HALDialect.h"
#include "iree/compiler/Dialect/HAL/IR/HALOps.h"
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Target/LLVMCPU/LLVMCPUTarget.cpp b/compiler/src/iree/compiler/Dialect/HAL/Target/LLVMCPU/LLVMCPUTarget.cpp
index 10a5960..9d1549e 100644
--- a/compiler/src/iree/compiler/Dialect/HAL/Target/LLVMCPU/LLVMCPUTarget.cpp
+++ b/compiler/src/iree/compiler/Dialect/HAL/Target/LLVMCPU/LLVMCPUTarget.cpp
@@ -11,7 +11,7 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
#include "iree-dialects/Dialect/LinalgTransform/LinalgTransformOps.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
#include "iree/compiler/Codegen/Utils/Utils.h"
#include "iree/compiler/Dialect/HAL/Target/LLVMCPU/Builtins/Device.h"
#include "iree/compiler/Dialect/HAL/Target/LLVMCPU/Builtins/Musl.h"
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Target/MetalSPIRV/MetalSPIRVTarget.cpp b/compiler/src/iree/compiler/Dialect/HAL/Target/MetalSPIRV/MetalSPIRVTarget.cpp
index d0d4417..390d8f7 100644
--- a/compiler/src/iree/compiler/Dialect/HAL/Target/MetalSPIRV/MetalSPIRVTarget.cpp
+++ b/compiler/src/iree/compiler/Dialect/HAL/Target/MetalSPIRV/MetalSPIRVTarget.cpp
@@ -7,7 +7,7 @@
#include "iree/compiler/Dialect/HAL/Target/MetalSPIRV/MetalSPIRVTarget.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Dialect/HAL/Target/MetalSPIRV/MSLToMetalLib.h"
#include "iree/compiler/Dialect/HAL/Target/MetalSPIRV/SPIRVToMSL.h"
#include "iree/compiler/Dialect/HAL/Target/TargetRegistry.h"
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Target/ROCM/ROCMTarget.cpp b/compiler/src/iree/compiler/Dialect/HAL/Target/ROCM/ROCMTarget.cpp
index d5187c7..6b66253 100644
--- a/compiler/src/iree/compiler/Dialect/HAL/Target/ROCM/ROCMTarget.cpp
+++ b/compiler/src/iree/compiler/Dialect/HAL/Target/ROCM/ROCMTarget.cpp
@@ -9,7 +9,7 @@
#include <mutex>
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
-#include "iree/compiler/Codegen/LLVMGPU/LLVMGPUPasses.h"
+#include "iree/compiler/Codegen/LLVMGPU/Passes.h"
#include "iree/compiler/Dialect/HAL/Target/TargetRegistry.h"
#include "iree/compiler/Utils/FlatbufferUtils.h"
#include "iree/schemas/rocm_executable_def_builder.h"
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Target/VMVX/VMVXTarget.cpp b/compiler/src/iree/compiler/Dialect/HAL/Target/VMVX/VMVXTarget.cpp
index 78b7e44..c8a88e3 100644
--- a/compiler/src/iree/compiler/Dialect/HAL/Target/VMVX/VMVXTarget.cpp
+++ b/compiler/src/iree/compiler/Dialect/HAL/Target/VMVX/VMVXTarget.cpp
@@ -8,7 +8,7 @@
#include "iree-dialects/Dialect/LinalgExt/IR/LinalgExtDialect.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
-#include "iree/compiler/Codegen/VMVX/VMVXPasses.h"
+#include "iree/compiler/Codegen/VMVX/Passes.h"
#include "iree/compiler/Dialect/Flow/IR/FlowOps.h"
#include "iree/compiler/Dialect/HAL/Target/TargetRegistry.h"
#include "iree/compiler/Dialect/VM/Conversion/ConversionTarget.h"
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Target/VulkanSPIRV/VulkanSPIRVTarget.cpp b/compiler/src/iree/compiler/Dialect/HAL/Target/VulkanSPIRV/VulkanSPIRVTarget.cpp
index bdd82ff..f008aaa 100644
--- a/compiler/src/iree/compiler/Dialect/HAL/Target/VulkanSPIRV/VulkanSPIRVTarget.cpp
+++ b/compiler/src/iree/compiler/Dialect/HAL/Target/VulkanSPIRV/VulkanSPIRVTarget.cpp
@@ -7,7 +7,7 @@
#include "iree/compiler/Dialect/HAL/Target/VulkanSPIRV/VulkanSPIRVTarget.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
#include "iree/compiler/Dialect/HAL/Target/TargetRegistry.h"
#include "iree/compiler/Dialect/Vulkan/IR/VulkanAttributes.h"
#include "iree/compiler/Dialect/Vulkan/IR/VulkanDialect.h"
diff --git a/compiler/src/iree/compiler/Dialect/HAL/Target/WebGPU/WebGPUTarget.cpp b/compiler/src/iree/compiler/Dialect/HAL/Target/WebGPU/WebGPUTarget.cpp
index c5085a8..0905020 100644
--- a/compiler/src/iree/compiler/Dialect/HAL/Target/WebGPU/WebGPUTarget.cpp
+++ b/compiler/src/iree/compiler/Dialect/HAL/Target/WebGPU/WebGPUTarget.cpp
@@ -7,8 +7,8 @@
#include "iree/compiler/Dialect/HAL/Target/WebGPU/WebGPUTarget.h"
#include "iree/compiler/Codegen/Dialect/IREECodegenDialect.h"
-#include "iree/compiler/Codegen/SPIRV/SPIRVPasses.h"
-#include "iree/compiler/Codegen/WGSL/WGSLPasses.h"
+#include "iree/compiler/Codegen/SPIRV/Passes.h"
+#include "iree/compiler/Codegen/WGSL/Passes.h"
#include "iree/compiler/Dialect/HAL/Target/TargetRegistry.h"
#include "iree/compiler/Dialect/HAL/Target/WebGPU/SPIRVToWGSL.h"
#include "iree/compiler/Dialect/HAL/Transforms/Passes.h"
diff --git a/compiler/src/iree/compiler/Dialect/VMVX/Transforms/BUILD.bazel b/compiler/src/iree/compiler/Dialect/VMVX/Transforms/BUILD.bazel
index 6edfb93..3e53b51 100644
--- a/compiler/src/iree/compiler/Dialect/VMVX/Transforms/BUILD.bazel
+++ b/compiler/src/iree/compiler/Dialect/VMVX/Transforms/BUILD.bazel
@@ -51,7 +51,6 @@
deps = [
":PassHeaders",
"//compiler/src/iree/compiler/Codegen/Common",
- "//compiler/src/iree/compiler/Codegen/Common:CommonPasses",
"//compiler/src/iree/compiler/Codegen/LLVMCPU",
"//compiler/src/iree/compiler/Codegen/Transforms",
"//compiler/src/iree/compiler/Codegen/Utils",
diff --git a/compiler/src/iree/compiler/Dialect/VMVX/Transforms/CMakeLists.txt b/compiler/src/iree/compiler/Dialect/VMVX/Transforms/CMakeLists.txt
index cd89c07..64c3449 100644
--- a/compiler/src/iree/compiler/Dialect/VMVX/Transforms/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Dialect/VMVX/Transforms/CMakeLists.txt
@@ -73,7 +73,6 @@
MLIRVectorDialect
MLIRVectorToSCF
iree::compiler::Codegen::Common
- iree::compiler::Codegen::Common::CommonPasses
iree::compiler::Codegen::LLVMCPU
iree::compiler::Codegen::Transforms
iree::compiler::Codegen::Utils
diff --git a/compiler/src/iree/compiler/Dialect/VMVX/Transforms/Passes.cpp b/compiler/src/iree/compiler/Dialect/VMVX/Transforms/Passes.cpp
index 86232ce..fe7d510 100644
--- a/compiler/src/iree/compiler/Dialect/VMVX/Transforms/Passes.cpp
+++ b/compiler/src/iree/compiler/Dialect/VMVX/Transforms/Passes.cpp
@@ -9,9 +9,9 @@
#include <memory>
#include "iree-dialects/Dialect/LinalgExt/Passes/Passes.h"
-#include "iree/compiler/Codegen/Common/CommonPasses.h"
-#include "iree/compiler/Codegen/LLVMCPU/LLVMCPUPasses.h"
-#include "iree/compiler/Codegen/VMVX/VMVXPasses.h"
+#include "iree/compiler/Codegen/Common/Passes.h"
+#include "iree/compiler/Codegen/LLVMCPU/Passes.h"
+#include "iree/compiler/Codegen/VMVX/Passes.h"
#include "iree/compiler/Dialect/HAL/Transforms/Passes.h"
#include "mlir/Conversion/AffineToStandard/AffineToStandard.h"
#include "mlir/Conversion/SCFToControlFlow/SCFToControlFlow.h"
diff --git a/compiler/src/iree/compiler/Modules/HAL/Inline/Conversion/HALToHALInline/test/CMakeLists.txt b/compiler/src/iree/compiler/Modules/HAL/Inline/Conversion/HALToHALInline/test/CMakeLists.txt
index 06c36b8..e69de29 100644
--- a/compiler/src/iree/compiler/Modules/HAL/Inline/Conversion/HALToHALInline/test/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Modules/HAL/Inline/Conversion/HALToHALInline/test/CMakeLists.txt
@@ -1,24 +0,0 @@
-################################################################################
-# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from #
-# compiler/src/iree/compiler/Modules/HAL/Inline/Conversion/HALToHALInline/test/BUILD.bazel#
-# #
-# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary #
-# CMake-only content. #
-# #
-# To disable autogeneration for this file entirely, delete this header. #
-################################################################################
-
-iree_add_all_subdirs()
-
-iree_lit_test_suite(
- NAME
- lit
- SRCS
- "buffer_ops.mlir"
- "buffer_view_ops.mlir"
- TOOLS
- FileCheck
- iree-opt
-)
-
-### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
