Merge google -> main (#8013) * fdeb04591 Merge pull request #8006 from not-jenni:main-to-google * 064ce933c Flip Identifier to StringAttr * ab66b4f30 Integrate LLVM at llvm/llvm-project@b5149f4e66a4 * 603f7a481 Integrate LLVM at llvm/llvm-project@37be74885946 * e8016e5a5 Integrate LLVM at llvm/llvm-project@11a46b174923 * a8a43826e Synchronize submodules with LLVM at llvm/llvm-project@3064dd8ccffc * 6f907859a Merge pull request #8003 from not-jenni:google * 4260a6261 Integrate LLVM at llvm/llvm-project@3064dd8ccffc * d0a228e7d Integrate LLVM at llvm/llvm-project@4c2aba999e54 * aec0a34c4 Integrate LLVM at llvm/llvm-project@cb9ccd38c55f * ee208bdc3 Integrate LLVM at llvm/llvm-project@862fffd8231c * 129ce7441 Integrate LLVM at llvm/llvm-project@127d95544164 * 16185fa35 Integrate LLVM at llvm/llvm-project@5527139302d9 * e9d9c3244 Integrate LLVM at llvm/llvm-project@249a5fb005ea * 8a3446001 Integrate LLVM at llvm/llvm-project@41454ab25645 * 09998baf4 Integrate LLVM at llvm/llvm-project@7d659c6ac741 * 3bbd7f386 Integrate LLVM at llvm/llvm-project@4943cda3988a * 44c9a30e7 Integrate LLVM at llvm/llvm-project@0c6979b2d64d * 7c4f1e6a4 Integrate LLVM at llvm/llvm-project@7128bb61fb59
diff --git a/docs/website/docs/reference/optimization-options.md b/docs/website/docs/reference/optimization-options.md new file mode 100644 index 0000000..b980938 --- /dev/null +++ b/docs/website/docs/reference/optimization-options.md
@@ -0,0 +1,54 @@ +# Optimization Options + +This page documents various supported flags for optimizing IREE programs. Each +is presented with its English name, flag to enable/disable, and default state. + +These flags can be passed to the: + +* `ireec` command line tool +* `extra_args=["--flag"]` argument to `iree.compiler.tools` Python wrappers +* In-process Python compiler API + `iree.compiler.transforms.ireec.CompilerOptions("--flag", "--flag2")` + constructor +* `ireeCompilerOptionsSetFlags()` compiler C API function + +## High level program optimizations + +### Constant evaluation (`--iree-opt-const-eval` (off)) + +Performs compile-time evaluation of any global initializers which produce +the initial values for global constants, storing the global directly in the +program as constant data. This extracts such constant program fragments and +recursively compiles them, using the runtime to evaluate the results. + +Note that this only has any effect on computations in module initializer +functions, not free-standing operations in the program which may produce +constant-derived results. See `--iree-opt-const-expr-hoisting` for options to +optimize these. + +### Constant expression hoisting (`--iree-opt-const-expr-hoisting` (off)) + +Identifies all trees of constant expressions in the program and uses a +heuristic to determine which would be profitable to hoist into global +initializers for evaluation at module load. Together with +`--iree-opt-const-eval`, this will convert eligible trees of expressions to +purely static data embedded in the module. + +The heuristic is currently relatively primitive, using static information to +disable hoisting of leaf operations which are metadata only (i.e. +broadcasts, etc) or are expected to fold away as part of operator fusion. +Notably, the current heuristic is likely to pessimize module size in the case of +complicated programs with trees of constant, large tensors. + +### Numeric precision reduction (`--iree-opt-numeric-precision-reduction` (off)) + +Analyzes program constant data and program flow to identify math operations +which can be safely evaluated with reduced precision (currently with a minimum +of 8bit integers but being extended to infer any bit depth) and inserts +appropriate casts. In conjunction with *Constant Expression Hoisting*, +*Constant Evaluation* and other automatic optimizations, this can produce +programs where large amounts (up to the whole) have had their numeric operations +and constant data rewritten to lower precision types. + +This feature is actively evolving and will be the subject of dedicated +documentation when ready.
diff --git a/docs/website/mkdocs.yml b/docs/website/mkdocs.yml index 1178135..a75eead 100644 --- a/docs/website/mkdocs.yml +++ b/docs/website/mkdocs.yml
@@ -117,6 +117,8 @@ - TensorFlow Lite: 'bindings/tensorflow-lite.md' - 'Extensions': - 'extensions/index.md' + - 'Reference': + - Optimization Options: 'reference/optimization-options.md' - 'Community': - 'community/index.md' - 'Blog':
diff --git a/iree/compiler/Dialect/Flow/Transforms/Passes.cpp b/iree/compiler/Dialect/Flow/Transforms/Passes.cpp index f1a7f30..805ce79 100644 --- a/iree/compiler/Dialect/Flow/Transforms/Passes.cpp +++ b/iree/compiler/Dialect/Flow/Transforms/Passes.cpp
@@ -95,6 +95,12 @@ transformOptions.buildConstEvalPassPipeline(pipeline); } + if (transformOptions.numericPrecisionReduction) { + pipeline.addPass(createInferNumericNarrowingPass()); + pipeline.addPass(createOptimizeNumericsPass()); + pipeline.addPass(createCleanupNumericNarrowingPass()); + } + FunctionLikeNest(pipeline) .addPass(mlir::createCanonicalizerPass) .addPass(mlir::createCSEPass);
diff --git a/iree/compiler/Dialect/Flow/Transforms/Passes.h b/iree/compiler/Dialect/Flow/Transforms/Passes.h index fb9854a..103d428 100644 --- a/iree/compiler/Dialect/Flow/Transforms/Passes.h +++ b/iree/compiler/Dialect/Flow/Transforms/Passes.h
@@ -30,6 +30,9 @@ // become the default. bool constExprHoisting = false; + // Enables passes to perform numeric precision reduction. + bool numericPrecisionReduction = false; + // Hook to populate a constant evaluation pass pipeline. If nullptr, then // no passes are added for constant evaluation. This must be injected in // because constant-evaluators can depend on the whole compiler, of which
diff --git a/iree/compiler/Dialect/Util/Transforms/FuseGlobals.cpp b/iree/compiler/Dialect/Util/Transforms/FuseGlobals.cpp index 49bcb07..7a5f900 100644 --- a/iree/compiler/Dialect/Util/Transforms/FuseGlobals.cpp +++ b/iree/compiler/Dialect/Util/Transforms/FuseGlobals.cpp
@@ -15,6 +15,7 @@ #include "llvm/ADT/EquivalenceClasses.h" #include "llvm/ADT/SmallVector.h" #include "llvm/Support/Debug.h" +#include "mlir/IR/AsmState.h" #include "mlir/IR/Matchers.h" #include "mlir/IR/PatternMatch.h" #include "mlir/Pass/Pass.h" @@ -73,6 +74,14 @@ } }; +static llvm::raw_ostream &operator<<(llvm::raw_ostream &os, + llvm::BitVector &bits) { + for (unsigned i = 0; i < bits.size(); ++i) { + os << (bits.test(i) ? "1" : "0"); + } + return os; +} + // Fuses globals that are always set to the same value into one. // // Example: @@ -111,17 +120,33 @@ // Note that we are only looking for stores within the same block - we // expect other canonicalizations to have moved stores into the same block // that are guaranteed to be on the same execution path. + // + // DenseMap<StringRef, llvm::BitVector> correlationMap; llvm::BitVector tempBits(globalTable.size()); for (auto callableOp : moduleOp.getOps<CallableOpInterface>()) { + LLVM_DEBUG(llvm::dbgs() + << "FuseGlobals: analyzing " << callableOp << ":\n"); for (auto &block : *callableOp.getCallableRegion()) { DenseMap<Value, SmallVector<IREE::Util::GlobalStoreOp>> valueStores; for (auto storeOp : block.getOps<IREE::Util::GlobalStoreOp>()) { auto &global = globalTable.globalMap[storeOp.global()]; + LLVM_DEBUG(llvm::dbgs() + << " - store #" << global.ordinal << ": " << storeOp + << "; candidate=" << global.isCandidate() << "\n"); if (!global.isCandidate()) continue; valueStores[storeOp.value()].push_back(storeOp); } for (auto valueStore : valueStores) { + LLVM_DEBUG({ + AsmState asmState(callableOp); + llvm::dbgs() << "= storing value "; + valueStore.first.printAsOperand(llvm::dbgs(), asmState); + llvm::dbgs() << ":\n"; + for (auto storeOp : valueStore.second) { + LLVM_DEBUG(llvm::dbgs() << " => @" << storeOp.global() << "\n"); + } + }); tempBits.reset(); for (auto storeOp : valueStore.second) { auto &global = globalTable.globalMap[storeOp.global()]; @@ -139,6 +164,43 @@ } } + // Resolve which globals are always set to the same value. + // This ensures that if @a is set to @b that @b is also set to @a. + // TODO(benvanik): find a better data structure that avoids the need for + // this cleanup step. We should be able to do this during construction. + for (auto it : correlationMap) { + auto globalName = it.first; + auto &correlationBits = it.second; + auto &global = globalTable.globalMap[globalName]; + llvm::BitVector tempBits = correlationBits; + for (auto ordinal : correlationBits.set_bits()) { + auto &otherGlobalName = globalTable.globalOrder[ordinal]; + if (otherGlobalName == globalName) continue; + auto &otherBits = correlationMap[otherGlobalName]; + tempBits &= otherBits; + } + if (!tempBits.test(global.ordinal)) { + // If the global we are analyzing isn't correlated with itself then we + // can't modify it at all. + tempBits.reset(); + } + correlationMap[globalName] = tempBits; + } + + LLVM_DEBUG({ + llvm::dbgs() << "FuseGlobals correlation maps:\n"; + for (auto it : correlationMap) { + auto globalName = it.first; + auto &correlationBits = it.second; + auto &global = globalTable.globalMap[globalName]; + llvm::dbgs() << "= #" << global.ordinal << " " << global.op.getName() + << " = " << correlationBits << ":\n"; + for (auto ordinal : correlationBits.set_bits()) { + llvm::dbgs() << " => " << globalTable.globalOrder[ordinal] << "\n"; + } + } + }); + // Build equivalence classes for each global, giving us nice clustered sets. // We could probably fold this with the step above but my head hurts. llvm::EquivalenceClasses<StringRef> ec;
diff --git a/iree/compiler/Dialect/Util/Transforms/test/fuse_globals.mlir b/iree/compiler/Dialect/Util/Transforms/test/fuse_globals.mlir index 08812e6..114b2f1 100644 --- a/iree/compiler/Dialect/Util/Transforms/test/fuse_globals.mlir +++ b/iree/compiler/Dialect/Util/Transforms/test/fuse_globals.mlir
@@ -66,3 +66,19 @@ // CHECK: return %[[VALUE0]], %[[VALUE1]] return %0, %1 : index, index } + +// ----- + +// CHECK: util.global private mutable @unfusableDivergent0 +util.global private mutable @unfusableDivergent0 : index +// CHECK: util.global private mutable @unfusableDivergent1 +util.global private mutable @unfusableDivergent1 : index +builtin.func @fn_a(%arg0: index) { + util.global.store %arg0, @unfusableDivergent0 : index + util.global.store %arg0, @unfusableDivergent1 : index + return +} +builtin.func @fn_b(%arg0: index) { + util.global.store %arg0, @unfusableDivergent0 : index + return +}
diff --git a/iree/compiler/Translation/IREEVM.cpp b/iree/compiler/Translation/IREEVM.cpp index a9eeb7f..5f16a90 100644 --- a/iree/compiler/Translation/IREEVM.cpp +++ b/iree/compiler/Translation/IREEVM.cpp
@@ -69,16 +69,21 @@ "IREE options for controlling high level optimizations"); binder.opt<bool>( - "iree-const-eval", constEval, + "iree-opt-const-eval", constEval, llvm::cl::desc("Enables eager evaluation of constants using the full " "compiler and runtime"), llvm::cl::cat(category)); binder.opt<bool>( - "iree-const-expr-hoisting", constExprHoisting, + "iree-opt-const-expr-hoisting", constExprHoisting, llvm::cl::desc( "Hoists the results of latent constant expressions into immutable " "global initializers for evaluation at program load"), llvm::cl::cat(category)); + binder.opt<bool>( + "iree-opt-numeric-precision-reduction", numericPrecisionReduction, + llvm::cl::desc( + "Reduces numeric precision to lower bit depths where possible"), + llvm::cl::cat(category)); } void buildIREEVMTransformPassPipeline( @@ -122,6 +127,8 @@ passManager.addPass(ConstEval::createJitGlobalsPass()); }; } + flowOptions.numericPrecisionReduction = + highLevelOptimizationOptions.numericPrecisionReduction; IREE::Flow::buildFlowTransformPassPipeline(passManager, flowOptions); IREE::Stream::TransformOptions streamOptions;
diff --git a/iree/compiler/Translation/IREEVM.h b/iree/compiler/Translation/IREEVM.h index dbf601f..550b454 100644 --- a/iree/compiler/Translation/IREEVM.h +++ b/iree/compiler/Translation/IREEVM.h
@@ -72,6 +72,9 @@ // and runtime. bool constEval = false; + // Optimizations to reduce numeric precision where it is safe to do so. + bool numericPrecisionReduction = false; + void bindOptions(OptionsBinder &binder); using FromFlags = OptionsFromFlags<HighLevelOptimizationOptions>; };