blob: b3c61697fb98d6ce8c74523a320dc3830aabcafa [file] [edit]
// Copyright 2022 The IREE Authors
//
// Licensed under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
#include "iree/compiler/Pipelines/Pipelines.h"
#include "iree/compiler/Bindings/Native/Transforms/Passes.h"
#include "iree/compiler/Bindings/TFLite/Transforms/Passes.h"
#include "iree/compiler/Dialect/Flow/Transforms/Passes.h"
#include "iree/compiler/Dialect/HAL/Transforms/Passes.h"
#include "iree/compiler/Dialect/Stream/Transforms/Passes.h"
#include "iree/compiler/Dialect/Util/Transforms/Passes.h"
#include "iree/compiler/Dialect/VM/Transforms/Passes.h"
#include "iree/compiler/DispatchCreation/Passes.h"
#include "iree/compiler/GlobalOptimization/Passes.h"
#include "iree/compiler/InputConversion/Common/Passes.h"
#include "iree/compiler/Modules/HAL/Inline/Transforms/Passes.h"
#include "iree/compiler/Modules/HAL/Loader/Transforms/Passes.h"
#include "iree/compiler/Preprocessing/Passes.h"
#include "iree/compiler/Utils/TracingUtils.h"
namespace mlir::iree_compiler {
static IREE::HAL::PipelinePhase
getHALPipelinePhase(IREEVMPipelinePhase phase,
IREE::HAL::PipelinePhase defaultPhase) {
switch (phase) {
default:
return defaultPhase;
case IREEVMPipelinePhase::ExecutableSources:
return IREE::HAL::PipelinePhase::ExecutableSources;
case IREEVMPipelinePhase::ExecutableConfigurations:
return IREE::HAL::PipelinePhase::ExecutableConfigurations;
case IREEVMPipelinePhase::ExecutableTargets:
return IREE::HAL::PipelinePhase::ExecutableTargets;
}
}
static IREEVMPipelinePhase
getIREEVMPipelinePhase(IREE::HAL::PipelinePhase phase) {
switch (phase) {
default:
return IREEVMPipelinePhase::HAL;
case IREE::HAL::PipelinePhase::ExecutableSources:
return IREEVMPipelinePhase::ExecutableSources;
case IREE::HAL::PipelinePhase::ExecutableConfigurations:
return IREEVMPipelinePhase::ExecutableConfigurations;
case IREE::HAL::PipelinePhase::ExecutableTargets:
return IREEVMPipelinePhase::ExecutableTargets;
}
}
IREEVMPipelineHooks::operator IREE::HAL::PipelineHooks() const {
IREE::HAL::PipelineHooks halHooks;
auto beforePhase = this->beforePhase;
halHooks.beforePhase = [beforePhase](IREE::HAL::PipelinePhase phase,
OpPassManager &passManager) {
if (beforePhase) {
beforePhase(getIREEVMPipelinePhase(phase), passManager);
}
};
auto afterPhase = this->afterPhase;
halHooks.afterPhase = [afterPhase](IREE::HAL::PipelinePhase phase,
OpPassManager &passManager) {
if (afterPhase) {
afterPhase(getIREEVMPipelinePhase(phase), passManager);
}
};
return halHooks;
}
void buildIREEPrecompileTransformPassPipeline(
const IREE::HAL::TargetRegistry &targetRegistry,
GlobalPipelineOptions pipelineOptions, BindingOptions bindingOptions,
InputDialectOptions inputOptions, PreprocessingOptions preprocessingOptions,
ParameterOptions parameterOptions,
GlobalOptimizationOptions globalOptimizationOptions,
DispatchCreationOptions dispatchCreationOptions,
SchedulingOptions schedulingOptions,
IREE::HAL::TargetOptions halTargetOptions, IREEVMPipelineHooks &hooks,
OpPassManager &passManager, IREEVMPipelinePhase compileFrom,
IREEVMPipelinePhase compileTo) {
// Input pipelines can result in changes to the exported functions and types
// and must run before generating bindings.
// After input processing, there should only be IREE legal types in
// signatures.
if (compileFrom < IREEVMPipelinePhase::Input) { // late-entry
auto inputType = inputOptions.parseInputTypeMnemonic();
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "Input");
if (hooks.beforePhase) {
hooks.beforePhase(IREEVMPipelinePhase::Input, passManager);
}
if (hooks.pipelineExtensions) {
hooks.pipelineExtensions->extendInputConversionPreprocessingPassPipeline(
passManager, inputType);
}
switch (inputType) {
case InputDialectOptions::Type::none:
break;
case InputDialectOptions::Type::auto_detect:
// Run the auto pipeline that chooses from plugins using module contents.
passManager.addPass(
InputConversion::createAutoInputConversionPipelinePass(
hooks.pipelineExtensions));
break;
case InputDialectOptions::Type::plugin: {
// Explicitly use a single plugin.
bool foundExtension = false;
if (hooks.pipelineExtensions) {
foundExtension =
hooks.pipelineExtensions->extendCustomInputConversionPassPipeline(
passManager, inputOptions.inputTypeMnemonic);
}
// We expect that callers properly validate supported extensions and that
// if a plugin advertises support, it actually provides it.
assert(foundExtension &&
"InputDialect::type::plugin extension not found");
if (!foundExtension) {
llvm::errs() << "internal error: InputDialect::type::plugin extension "
"not found ("
<< inputOptions.inputTypeMnemonic << ")\n";
}
break;
}
}
InputConversion::TransformOptions inputTransformOptions;
inputTransformOptions.options = inputOptions;
InputConversion::buildCommonInputConversionPassPipeline(
passManager, inputTransformOptions);
if (hooks.afterPhase) {
hooks.afterPhase(IREEVMPipelinePhase::Input, passManager);
}
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "Input");
}
if (compileTo == IREEVMPipelinePhase::Input) {
return; // early-exit
}
// Now that inputs are legalized, generate wrapper for entry functions.
if (compileFrom < IREEVMPipelinePhase::ABI) { // late-entry
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "ABI");
if (hooks.beforePhase) {
hooks.beforePhase(IREEVMPipelinePhase::ABI, passManager);
}
IREE::ABI::InvocationOptions invocationOptions;
invocationOptions.invocationModel =
schedulingOptions.executionModel ==
SchedulingOptions::ExecutionModel::AsyncExternal
? IREE::ABI::InvocationModel::CoarseFences
: IREE::ABI::InvocationModel::Sync;
if (bindingOptions.native) {
IREE::ABI::buildTransformPassPipeline(passManager, invocationOptions);
}
if (bindingOptions.tflite) {
IREE::TFLite::buildTransformPassPipeline(passManager);
}
if (hooks.afterPhase) {
hooks.afterPhase(IREEVMPipelinePhase::ABI, passManager);
}
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "ABI");
}
if (compileTo == IREEVMPipelinePhase::ABI) {
return; // early-exit
}
// If the user specified a set of target devices we attach them to the module
// IR so that they are available for all passes that may want to use this
// information. If trying to compile in a generic mode the user should omit
// specifying targets.
IREE::HAL::AssignmentOptions halAssignmentOptions;
halAssignmentOptions.legacyTargetBackends =
halTargetOptions.legacyTargetBackends;
halAssignmentOptions.targetDevices = halTargetOptions.targetDevices;
halAssignmentOptions.defaultDevice = halTargetOptions.defaultDevice;
IREE::HAL::buildHALDeviceAssignmentPassPipeline(passManager, targetRegistry,
halAssignmentOptions);
GlobalOptimization::TransformOptions globalTransformOptions;
globalTransformOptions.parameterImportPaths = parameterOptions.importPaths;
globalTransformOptions.parameterImportKeys = parameterOptions.importKeys;
globalTransformOptions.parameterImportMaximumSize =
parameterOptions.importMaximumSize;
globalTransformOptions.parameterExportPath = parameterOptions.exportPath;
globalTransformOptions.parameterExportMinimumSize =
parameterOptions.exportMinimumSize;
globalTransformOptions.parameterSplatPath = parameterOptions.splatPath;
globalTransformOptions.aggressiveTransposePropagation =
globalOptimizationOptions.aggressiveTransposePropagation;
globalTransformOptions.propagateTransposesThroughConv =
globalOptimizationOptions.propagateTransposesThroughConv;
globalTransformOptions.sinkTransposeThroughPad =
globalOptimizationOptions.sinkTransposeThroughPad;
globalTransformOptions.outerDimConcat =
globalOptimizationOptions.outerDimConcat;
// The pipeline option has higher priority.
globalTransformOptions.dataTiling = globalOptimizationOptions.dataTiling;
if (pipelineOptions.dataTiling) {
globalTransformOptions.dataTiling = false;
}
globalTransformOptions.constEval = globalOptimizationOptions.constEval;
globalTransformOptions.numericPrecisionReduction =
globalOptimizationOptions.numericPrecisionReduction;
globalTransformOptions.stripAssertions =
globalOptimizationOptions.stripAssertions;
globalTransformOptions.generalizeMatmul =
globalOptimizationOptions.generalizeMatmul;
globalTransformOptions.useIm2colForConvs =
globalOptimizationOptions.useIm2colForConvs;
globalTransformOptions.constExprHoisting = pipelineOptions.constExprHoisting;
globalTransformOptions.constExprMaxSizeIncreaseThreshold =
pipelineOptions.constExprMaxSizeIncreaseThreshold;
// Enable const-eval via hook. For debug builds, we assert if enabled
// without a hook. For release, we just silently skip enabling const-eval.
if (globalOptimizationOptions.constEval) {
assert(hooks.buildConstEvalPassPipelineCallback &&
"if const-eval is enabled the buildConstEvalPassPipelineCallback "
"hook must be enabled");
}
if (globalOptimizationOptions.constEval &&
hooks.buildConstEvalPassPipelineCallback) {
globalTransformOptions.buildConstEvalPassPipeline =
hooks.buildConstEvalPassPipelineCallback;
}
switch (schedulingOptions.executionModel) {
case SchedulingOptions::ExecutionModel::HostOnly:
// No flow/stream processing (implies no tensors).
break;
default:
if (compileFrom < IREEVMPipelinePhase::Preprocessing) { // late-entry.
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "Preprocessing");
if (hooks.beforePhase) {
hooks.beforePhase(IREEVMPipelinePhase::Preprocessing, passManager);
}
Preprocessing::buildPreprocessingPassPipeline(
passManager, preprocessingOptions, hooks.pipelineExtensions);
if (hooks.afterPhase) {
hooks.afterPhase(IREEVMPipelinePhase::Preprocessing, passManager);
}
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "Preprocessing");
}
if (compileTo == IREEVMPipelinePhase::Preprocessing) {
return; // early-exit
}
if (compileFrom < IREEVMPipelinePhase::GlobalOptimization) { // late-entry
// This pass pipeline recursively invokes the compiler if constEval is
// enabled. In that case, we have to be careful to not emit unbalanced
// trace frames:
// begin 'GlobalOptimization'
// begin 'Input'
// end 'Input'
// begin 'GlobalOptimization' <-- unbalanced! Use a different name.
// end 'GlobalOptimization'
// ...
// end 'GlobalOptimization'
if (globalOptimizationOptions.constEval) {
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "GlobalOptimizationConst");
} else {
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "GlobalOptimization");
}
if (hooks.beforePhase) {
hooks.beforePhase(IREEVMPipelinePhase::GlobalOptimization, passManager);
}
GlobalOptimization::buildGlobalOptimizationPassPipeline(
passManager, globalTransformOptions);
if (hooks.afterPhase) {
hooks.afterPhase(IREEVMPipelinePhase::GlobalOptimization, passManager);
}
if (globalOptimizationOptions.constEval) {
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "GlobalOptimizationConst");
} else {
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "GlobalOptimization");
}
}
if (compileTo == IREEVMPipelinePhase::GlobalOptimization) {
return; // early-exit
}
break;
}
}
void buildIREEVMTransformPassPipeline(
const IREE::HAL::TargetRegistry &targetRegistry,
GlobalPipelineOptions pipelineOptions, BindingOptions bindingOptions,
InputDialectOptions inputOptions, PreprocessingOptions preprocessingOptions,
ParameterOptions parameterOptions,
GlobalOptimizationOptions globalOptimizationOptions,
DispatchCreationOptions dispatchCreationOptions,
SchedulingOptions schedulingOptions,
IREE::HAL::TargetOptions halTargetOptions,
IREE::VM::TargetOptions vmTargetOptions, IREEVMPipelineHooks &hooks,
OpPassManager &passManager, IREEVMPipelinePhase compileFrom,
IREEVMPipelinePhase compileTo) {
buildIREEPrecompileTransformPassPipeline(
targetRegistry, pipelineOptions, bindingOptions, inputOptions,
preprocessingOptions, parameterOptions, globalOptimizationOptions,
dispatchCreationOptions, schedulingOptions, halTargetOptions, hooks,
passManager, compileFrom, compileTo);
if (compileTo <= IREEVMPipelinePhase::GlobalOptimization) {
return; // early-exit
}
IREE::Stream::TransformOptions streamOptions;
// TODO(benvanik): find a way to share the enums w/o circular deps.
streamOptions.initializationMode =
(IREE::Stream::InitializationMode)schedulingOptions.initializationMode;
streamOptions.optimizeBindings = schedulingOptions.optimizeBindings;
streamOptions.dumpStatisticsFormat =
(IREE::Stream::DumpOutputFormat)schedulingOptions.dumpStatisticsFormat;
streamOptions.dumpStatisticsFile = schedulingOptions.dumpStatisticsFile;
// Set parameter encoder options. These are mapped to
// SplitParameterEncoderPassOptions when the pass is created in
// Stream/Transforms/Passes.cpp.
if (!parameterOptions.encoderOutputFile.empty()) {
streamOptions.parameterEncoderMode =
(IREE::Stream::ParameterEncoderMode)parameterOptions.encoderMode;
streamOptions.parameterEncoderOutputFile =
parameterOptions.encoderOutputFile;
streamOptions.parameterEncoderOutputScope =
parameterOptions.encoderOutputScope;
}
switch (schedulingOptions.executionModel) {
case SchedulingOptions::ExecutionModel::HostOnly:
// No flow/stream processing (implies no tensors).
break;
default:
DispatchCreation::TransformOptions dispatchTransformOptions;
dispatchTransformOptions.enableAggressiveFusion =
dispatchCreationOptions.enableAggressiveFusion;
dispatchTransformOptions.enableFuseMultiUse =
dispatchCreationOptions.enableFuseMultiUse;
// The pipeline option has higher priority.
dispatchTransformOptions.dataTiling = dispatchCreationOptions.dataTiling;
if (pipelineOptions.dataTiling) {
dispatchTransformOptions.dataTiling = true;
}
if (dispatchTransformOptions.dataTiling &&
globalOptimizationOptions.dataTiling) {
#ifndef NDEBUG
llvm::reportFatalUsageError(
"Invalid configuration: data-tiling cannot be enabled in both "
"global optimization phase and dispatch creation phase.");
#endif
dispatchTransformOptions.dataTiling = false;
}
dispatchTransformOptions.enableSplitReduction =
dispatchCreationOptions.enableSplitReduction;
dispatchTransformOptions.splitReductionLowParallelism =
dispatchCreationOptions.splitReductionLowParallelism;
dispatchTransformOptions.enableAggressiveReshapeMovement =
dispatchCreationOptions.enableAggressiveReshapeMovement;
dispatchTransformOptions.enablePadHandling =
dispatchCreationOptions.enablePadHandling;
dispatchTransformOptions.enableFusePaddingIntoLinalgConsumerOps =
dispatchCreationOptions.enableFusePaddingIntoLinalgConsumerOps;
dispatchTransformOptions.constExprMaxSizeIncreaseThreshold =
pipelineOptions.constExprMaxSizeIncreaseThreshold;
dispatchTransformOptions.constExprHoisting =
pipelineOptions.constExprHoisting;
if (compileFrom < IREEVMPipelinePhase::DispatchCreation) { // late-entry
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "DispatchCreation");
if (hooks.beforePhase) {
hooks.beforePhase(IREEVMPipelinePhase::DispatchCreation, passManager);
}
DispatchCreation::buildDispatchCreationPassPipeline(
passManager, dispatchTransformOptions);
if (hooks.afterPhase) {
hooks.afterPhase(IREEVMPipelinePhase::DispatchCreation, passManager);
}
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "DispatchCreation");
}
if (compileTo == IREEVMPipelinePhase::DispatchCreation) {
return; // early-exit
}
IREE::Flow::TransformOptions flowOptions;
if (compileFrom < IREEVMPipelinePhase::Flow) { // late-entry
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "Flow");
if (hooks.beforePhase) {
hooks.beforePhase(IREEVMPipelinePhase::Flow, passManager);
}
IREE::Flow::buildFlowTransformPassPipeline(passManager, flowOptions);
if (hooks.afterPhase) {
hooks.afterPhase(IREEVMPipelinePhase::Flow, passManager);
}
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "Flow");
}
if (compileTo == IREEVMPipelinePhase::Flow) {
return; // early-exit
}
if (compileFrom < IREEVMPipelinePhase::Stream) { // late-entry
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "Stream");
if (hooks.beforePhase) {
hooks.beforePhase(IREEVMPipelinePhase::Stream, passManager);
}
IREE::Stream::buildStreamTransformPassPipeline(passManager,
streamOptions);
if (hooks.afterPhase) {
hooks.afterPhase(IREEVMPipelinePhase::Stream, passManager);
}
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "Stream");
}
if (compileTo == IREEVMPipelinePhase::Stream) {
return; // early-exit
}
break;
}
IREE::HAL::PipelinePhase halCompileFrom =
getHALPipelinePhase(compileFrom, IREE::HAL::PipelinePhase::Start);
IREE::HAL::PipelinePhase halCompileTo =
getHALPipelinePhase(compileTo, IREE::HAL::PipelinePhase::End);
if (compileFrom < IREEVMPipelinePhase::HAL) { // late-entry
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "HAL");
if (hooks.beforePhase) {
hooks.beforePhase(IREEVMPipelinePhase::HAL, passManager);
}
switch (schedulingOptions.executionModel) {
case SchedulingOptions::ExecutionModel::HostOnly:
// No HAL required.
break;
default:
case SchedulingOptions::ExecutionModel::AsyncInternal:
case SchedulingOptions::ExecutionModel::AsyncExternal:
IREE::HAL::buildHALTransformPassPipeline(passManager, targetRegistry,
halTargetOptions, hooks,
halCompileFrom, halCompileTo);
break;
case SchedulingOptions::ExecutionModel::InlineStatic:
IREE::HAL::Inline::buildHALInlineStaticTransformPassPipeline(
passManager, targetRegistry, halTargetOptions);
break;
case SchedulingOptions::ExecutionModel::InlineDynamic:
IREE::HAL::Loader::buildHALInlineDynamicTransformPassPipeline(
passManager, targetRegistry, halTargetOptions);
break;
}
if (hooks.afterPhase) {
hooks.afterPhase(IREEVMPipelinePhase::HAL, passManager);
}
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "HAL");
}
if (compileTo == IREEVMPipelinePhase::HAL ||
halCompileTo != IREE::HAL::PipelinePhase::End) {
return; // early-exit
}
if (compileFrom < IREEVMPipelinePhase::VM) { // late-entry
IREE_TRACE_ADD_BEGIN_FRAME_PASS(passManager, "VM");
if (hooks.beforePhase) {
hooks.beforePhase(IREEVMPipelinePhase::VM, passManager);
}
IREE::VM::buildVMTransformPassPipeline(passManager, vmTargetOptions);
if (hooks.afterPhase) {
hooks.afterPhase(IREEVMPipelinePhase::VM, passManager);
}
IREE_TRACE_ADD_END_FRAME_PASS(passManager, "VM");
}
if (compileTo == IREEVMPipelinePhase::VM) {
return; // early-exit
}
}
void buildDefaultIREEVMTransformPassPipeline(OpPassManager &passManager) {
// Note that the production compiler will provide hooks here that enable
// additional, whole-program related features, whereas this pipeline will
// only use the defaults. In practice, this means that things like const
// jitting are not supported by this pipeline (it would create a circular
// dependency at this point in the compiler to also depend on the compiler).
static IREEVMPipelineHooks defaultHooks;
// Since a JIT hook cannot be provided in such a default pipeline, we
// force disable const eval, which relies on the JIT.
auto highLevelOptimizations = GlobalOptimizationOptions::FromFlags::get();
highLevelOptimizations.constEval = false;
buildIREEVMTransformPassPipeline(
IREE::HAL::TargetRegistry::getGlobal(),
GlobalPipelineOptions::FromFlags::get(), BindingOptions::FromFlags::get(),
InputDialectOptions::FromFlags::get(),
PreprocessingOptions::FromFlags::get(),
ParameterOptions::FromFlags::get(), highLevelOptimizations,
DispatchCreationOptions::FromFlags::get(),
SchedulingOptions::FromFlags::get(),
IREE::HAL::TargetOptions::FromFlags::get(),
IREE::VM::TargetOptions::FromFlags::get(), defaultHooks, passManager);
}
void registerIREEVMTransformPassPipeline() {
PassPipelineRegistration<> transformPassPipeline(
"iree-transformation-pipeline",
"Runs the full IREE input to VM transformation pipeline",
[](OpPassManager &passManager) {
buildDefaultIREEVMTransformPassPipeline(passManager);
});
}
} // namespace mlir::iree_compiler