| // Copyright 2023 The IREE Authors |
| // |
| // Licensed under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| |
| #ifndef IREE_COMPILER_GLOBALOPTIMIZATION_PASSES |
| #define IREE_COMPILER_GLOBALOPTIMIZATION_PASSES |
| |
| include "mlir/Pass/PassBase.td" |
| |
| def CleanupNumericNarrowingPass : |
| Pass<"iree-global-opt-cleanup-numeric-narrowing", ""> { |
| let summary = "Cleans up any numeric narrowing ops inserted by iree-global-opt-infer-numeric-narrowing."; |
| } |
| |
| def ConvertBatchMatmulToMatmulPass : |
| Pass<"iree-global-opt-convert-batch-matmul-to-matmul", ""> { |
| let summary = "Converts batch_matmul to reshape + matmul + reshape."; |
| let dependentDialects = [ |
| "mlir::arith::ArithDialect", |
| "mlir::linalg::LinalgDialect", |
| "mlir::tensor::TensorDialect", |
| ]; |
| } |
| |
| def ConvertConv2DToImg2ColPass : |
| Pass<"iree-global-opt-convert-conv2d-to-img2col", ""> { |
| let summary = "Convert linalg convolution ops to matmul img2col based implementation"; |
| let dependentDialects = [ |
| "mlir::linalg::LinalgDialect", |
| "mlir::tensor::TensorDialect", |
| ]; |
| } |
| |
| def ConvertStridedContractionToContractionPass: |
| Pass<"iree-global-opt-convert-strided-contraction-to-contraction", ""> { |
| let summary = "Factors out an extract_slice from contraction-like ops with strided inputs."; |
| } |
| |
| def DecomposeConcatPass : |
| Pass<"iree-global-opt-decompose-concat", ""> { |
| let summary = "Decomposes concatenations into a destination and a sequence of slice inserts."; |
| let options = [ |
| Option<"enableConcatTransposition", "enable-concat-transposition", "bool", |
| /*default=*/"false", "Allows transposing concatenations such that " |
| "they occur on the inner most dims.">, |
| ]; |
| } |
| |
| def DemoteContractionInputsPass |
| : Pass<"iree-global-opt-demote-contraction-inputs", ""> { |
| let summary = "Demotes inputs (LHS, RHS) of linalg matmul-like ops from f32 " |
| "to a narrower type."; |
| let options = |
| [Option<"demoteType", "type", |
| "mlir::iree_compiler::GlobalOptimization::DemoteType", |
| /*default=*/"", "Select the target type for demotion.", |
| [{::llvm::cl::values( |
| clEnumValN(mlir::iree_compiler::GlobalOptimization::DemoteType::F16, |
| "f16", |
| "Demote to f16."), |
| clEnumValN(mlir::iree_compiler::GlobalOptimization::DemoteType::BF16, |
| "bf16", |
| "Demote to bf16.") |
| )}]>, |
| Option<"demoteOperation", "operation", |
| "mlir::iree_compiler::GlobalOptimization::DemoteOperation", |
| /*default=*/ |
| "mlir::iree_compiler::GlobalOptimization::DemoteOperation::All", |
| "Select the type of contraction ops to demote.", |
| [{::llvm::cl::values( |
| clEnumValN(mlir::iree_compiler::GlobalOptimization::DemoteOperation::All, |
| "all", |
| "Demote all contraction ops."), |
| clEnumValN(mlir::iree_compiler::GlobalOptimization::DemoteOperation::Conv, |
| "conv", |
| "Only demote convolution ops."), |
| clEnumValN(mlir::iree_compiler::GlobalOptimization::DemoteOperation::Matmul, |
| "matmul", |
| "Only demote matmul ops."), |
| clEnumValN(mlir::iree_compiler::GlobalOptimization::DemoteOperation::None, |
| "none", |
| "Demote no contraction ops.") |
| )}]>, |
| ]; |
| } |
| |
| def DetachElementwiseFromNamedOpsPass : |
| Pass<"iree-global-opt-detach-elementwise-from-named-ops", ""> { |
| let summary = "Detaches elementwise ops from named Linalg ops."; |
| } |
| |
| def EraseUnusedLinalgOperandsPass : |
| Pass<"iree-global-opt-erase-unused-linalg-operands", "mlir::ModuleOp"> { |
| let summary = "Erases unused linalg operand and remove dead code."; |
| } |
| |
| def ExpandTensorShapesPass : |
| Pass<"iree-global-opt-expand-tensor-shapes", "mlir::ModuleOp"> { |
| let summary = "Expands tensor shape dimensions into SSA values across the program."; |
| } |
| |
| def PropagateAffineQuantizationPass : |
| InterfacePass<"iree-global-opt-propagate-affine-quantization", "mlir::FunctionOpInterface"> { |
| let summary = "Propagates affine quantization through adjacent tensor operations."; |
| let description = [{ |
| Moves `iree_linalg_ext.quantize_affine` before a producer-side |
| `tensor.collapse_shape`, and moves a consumer-side `tensor.expand_shape`, |
| or a zero `tensor.pad` whose padded dimensions do not index the scale and |
| whose zero point is per-tensor, before `iree_linalg_ext.dequantize_affine`. |
| These shape operations then run on quantized values. A consumer-side |
| `linalg.transpose` is fused into the dequantize's output map instead. When |
| a contraction consumes the result, the dequantize becomes its direct |
| producer so the integer contraction rewrite can match it. |
| }]; |
| } |
| |
| def ConvertQDQToIntegerMathPass : |
| InterfacePass<"iree-global-opt-convert-qdq-to-integer-math", "mlir::FunctionOpInterface"> { |
| let summary = "Rewrites contractions over dequantized operands into integer contractions."; |
| let description = [{ |
| Matches a contraction or convolution whose inputs are produced by |
| `iree_linalg_ext.dequantize_affine` and rewrites it into an integer |
| contraction plus a zero point correction epilogue: |
| |
| C = sA*sB*(D - zB*RA - zA*RB + N*zA*zB) |
| |
| Matches a plain floating-point multiply-add body with a zero init. Integer |
| reduction dimensions must have positive static extents whose products and |
| zero point corrections fit in signed i32. Dimensions along which a |
| quantization parameter varies remain floating-point reductions over partial |
| integer contractions. Unsupported cases retain the original computation. |
| |
| Scaling and partial-result accumulation use at least f32, preserving wider |
| scale and output types, before conversion to the output type. |
| |
| Numerical stability is best effort. Reassociation changes floating-point |
| rounding; no fixed error bound or preservation of signed zero, NaNs, |
| infinities, or overflow/underflow behavior is guaranteed. Disable this |
| transformation when reference floating-point semantics are required with |
| `--iree-global-opt-enable-qdq-to-integer-math=false`. |
| }]; |
| } |
| |
| def FuseDequantizationMatmulPass: |
| InterfacePass<"iree-global-opt-fuse-dequantization-matmul", "mlir::FunctionOpInterface"> { |
| let summary = "Fuses dequantization and matmul linalg.generic ops."; |
| } |
| |
| def GeneralizeLinalgNamedOpsPass : |
| InterfacePass<"iree-global-opt-generalize-linalg-named-ops", "mlir::FunctionOpInterface"> { |
| let summary = "Convert some Linalg named ops into linalg.generics."; |
| let options = [ |
| Option<"enableGeneralizeMatmul", "enable-generalize-matmul", "bool", |
| /*default=*/"false", "Convert linalg named matmul ops to generic ops.">, |
| ]; |
| } |
| |
| def InferNumericNarrowingPass : |
| Pass<"iree-global-opt-infer-numeric-narrowing", ""> { |
| let summary = "Infers and inserts util.numeric.optional_narrow ops at points that may be beneficial."; |
| } |
| |
| def MaterializeHomogeneousEncodingsPass : |
| Pass<"iree-global-opt-materialize-homogeneous-encodings", "mlir::ModuleOp"> { |
| let summary = "Materializes logical encodings to physical encodings if there is a single device target."; |
| } |
| |
| def OptimizeNumericsPass : |
| Pass<"iree-global-opt-optimize-numerics", ""> { |
| let summary = "Optimizes numerics given annotations added via iree-global-opt-infer-numeric-narrowing."; |
| } |
| |
| def PropagateLinalgTransposePass : |
| InterfacePass<"iree-global-opt-propagate-linalg-transpose", "mlir::FunctionOpInterface"> { |
| let summary = "Propagates linalg.transpose through a restricted set of ops."; |
| let options = [ |
| Option<"enableAggressivePropagation", "enable-aggressive-propagation", "bool", |
| /*default=*/"false", "Enable aggressive propagation to named ops.">, |
| Option<"testSinkingOnly", "test-sinking-only", "bool", /*default=*/"false", |
| "Flag used for lit-testing sinking patterns only. Not for general usage">, |
| Option<"testBubblingOnly", "test-bubbling-only", "bool", /*default=*/"false", |
| "Flag used for lit-testing bubbling patterns only. Not for general usage">, |
| Option<"enableAttentionVTranspose", "enable-attention-v-transpose", "bool", |
| /*default=*/"true", "Enable transposition of attention v operand">, |
| Option<"enableConvolutionPropagation", "enable-aggressive-propagation-through-conv", "bool", |
| /*default=*/"false", "enable propagation through convolutions">, |
| Option<"enableEdgeReshapePropagation", "enable-edge-reshape-propagation", "bool", |
| /*default=*/"false", "Enable propagation of reshapes on the edges of the program">, |
| Option<"enableSinkTransposeThroughPad", "enable-sink-transpose-through-pad", "bool", |
| /*default=*/"false", "Enable sinking transpose through pad operations">, |
| ]; |
| } |
| |
| def LinalgQuantizedConvToConvPass |
| : InterfacePass<"iree-global-opt-quantized-conv-to-conv", "mlir::FunctionOpInterface"> { |
| let summary = "lower quantized_conv to conv"; |
| } |
| |
| def LinalgQuantizedMatmulToMatmulPass |
| : InterfacePass<"iree-global-opt-quantized-matmul-to-matmul", "mlir::FunctionOpInterface"> { |
| let summary = "lower quantized_matmul to matmul"; |
| } |
| |
| def RaiseSpecialOpsPass : |
| Pass<"iree-global-opt-raise-special-ops", ""> { |
| let summary = "Raises special ops like softmax to the high level linalg.ext representation."; |
| } |
| |
| def RemoveZeroExtentTensorsPass : |
| InterfacePass<"iree-global-opt-remove-zero-extent-tensors", "mlir::FunctionOpInterface"> { |
| let summary = "Removes tensors that have 0-extents."; |
| } |
| |
| def SimplifyPackUnpackPass : Pass<"iree-global-opt-simplify-pack-unpack", ""> { |
| let summary = "Simplifies tensor pack and unpack ops."; |
| } |
| |
| def GlobalLoopInvariantCodeMotionPass : InterfacePass<"iree-global-opt-loop-invariant-code-motion", "mlir::FunctionOpInterface"> { |
| let summary = "Hoist loop invariants out of loops with zero-trip-check."; |
| } |
| |
| def DataLayoutPropagationPass : InterfacePass<"iree-global-opt-data-layout-propagation", "mlir::FunctionOpInterface"> { |
| let summary = "Propagate pack/unpack ops across other ops to improve fusion"; |
| } |
| |
| def WarnOnUninitializedValuesPass : InterfacePass<"iree-global-opt-warn-on-uninitialized-values", "mlir::FunctionOpInterface"> { |
| let summary = "Catch some uses of uninitialized values"; |
| } |
| |
| #endif // IREE_COMPILER_GLOBALOPTIMIZATION_PASSES |
