| // Copyright 2021 The IREE Authors | |
| // | |
| // Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| #include <utility> | |
| #include "iree/compiler/Dialect/Util/IR/UtilDialect.h" | |
| #include "iree/compiler/Dialect/Util/IR/UtilOps.h" | |
| #include "iree/compiler/Dialect/Util/Transforms/Passes.h" | |
| #include "iree/compiler/Dialect/Util/Transforms/Patterns.h" | |
| #include "iree/compiler/Utils/IntegerSet.h" | |
| #include "llvm/ADT/BreadthFirstIterator.h" | |
| #include "llvm/ADT/STLExtras.h" | |
| #include "llvm/ADT/TypeSwitch.h" | |
| #include "llvm/Support/Debug.h" | |
| #include "mlir/Dialect/ControlFlow/IR/ControlFlowOps.h" | |
| #include "mlir/Dialect/SCF/IR/SCF.h" | |
| #include "mlir/IR/Attributes.h" | |
| #include "mlir/IR/Builders.h" | |
| #include "mlir/IR/BuiltinTypes.h" | |
| #include "mlir/IR/Diagnostics.h" | |
| #include "mlir/IR/Dominance.h" | |
| #include "mlir/IR/IRMapping.h" | |
| #include "mlir/Pass/Pass.h" | |
| #define DEBUG_TYPE "iree-util-propagate-subranges" | |
| namespace mlir::iree_compiler::IREE::Util { | |
| #define GEN_PASS_DEF_PROPAGATESUBRANGESPASS | |
| #include "iree/compiler/Dialect/Util/Transforms/Passes.h.inc" | |
| namespace { | |
| // This pass is paired with the subrange type. Any type implementing the | |
| // interface can be used. | |
| static bool isResourceType(Type type) { | |
| return llvm::isa<IREE::Util::SubrangeTypeInterface>(type); | |
| } | |
| //===----------------------------------------------------------------------===// | |
| // Global handling | |
| //===----------------------------------------------------------------------===// | |
| struct ExpandedGlobal { | |
| IREE::Util::GlobalOp resourceOp; | |
| IREE::Util::GlobalOp resourceSizeOp; | |
| IREE::Util::GlobalOp subrangeOffsetOp; | |
| IREE::Util::GlobalOp subrangeLengthOp; | |
| }; | |
| using ExpandedGlobalMap = DenseMap<StringRef, ExpandedGlobal>; | |
| // Expands each !stream.resource global in |rootOp| to have a matching | |
| // parent resource size and subrange range. Does not behave optimally if there | |
| // already exist offset globals as duplicates will get added and we'll need to | |
| // rely on global fusion to get rid of them. Note that this only expands globals | |
| // and does not yet update use sites - we just need the ops to reference. | |
| static ExpandedGlobalMap expandResourceGlobals(Operation *rootOp, | |
| SymbolTable &symbolTable) { | |
| ExpandedGlobalMap expandedGlobals; | |
| // Gather all of the resource globals in the root. | |
| for (auto ®ion : rootOp->getRegions()) { | |
| for (auto globalOp : region.getOps<IREE::Util::GlobalOp>()) { | |
| if (!isResourceType(globalOp.getType())) | |
| continue; | |
| expandedGlobals[globalOp.getName()].resourceOp = globalOp; | |
| } | |
| } | |
| // Expand each global by adding the offset right next to it. | |
| auto indexType = IndexType::get(rootOp->getContext()); | |
| for (auto &it : expandedGlobals) { | |
| auto &global = it.second; | |
| OpBuilder builder(global.resourceOp); | |
| builder.setInsertionPointAfter(global.resourceOp); | |
| auto sizeName = (global.resourceOp.getName() + "__storage_size").str(); | |
| auto sizeOp = builder.create<IREE::Util::GlobalOp>( | |
| global.resourceOp.getLoc(), sizeName, | |
| /*isMutable=*/true, indexType); | |
| sizeOp.setVisibility(global.resourceOp.getVisibility()); | |
| symbolTable.insert(sizeOp); | |
| global.resourceSizeOp = sizeOp; | |
| auto offsetName = (global.resourceOp.getName() + "__offset").str(); | |
| auto offsetOp = builder.create<IREE::Util::GlobalOp>( | |
| global.resourceOp.getLoc(), offsetName, | |
| /*isMutable=*/true, indexType); | |
| offsetOp.setVisibility(global.resourceOp.getVisibility()); | |
| symbolTable.insert(offsetOp); | |
| global.subrangeOffsetOp = offsetOp; | |
| auto lengthName = (global.resourceOp.getName() + "__length").str(); | |
| auto lengthOp = builder.create<IREE::Util::GlobalOp>( | |
| global.resourceOp.getLoc(), lengthName, | |
| /*isMutable=*/true, indexType); | |
| lengthOp.setVisibility(global.resourceOp.getVisibility()); | |
| symbolTable.insert(lengthOp); | |
| global.subrangeLengthOp = lengthOp; | |
| } | |
| return expandedGlobals; | |
| } | |
| //===----------------------------------------------------------------------===// | |
| // Structural IR rewriting patterns | |
| //===----------------------------------------------------------------------===// | |
| // Returns true if an operands or results of |op| use !stream.resources. | |
| static bool usesResources(Operation *op) { | |
| return llvm::any_of(op->getOperandTypes(), isResourceType) || | |
| llvm::any_of(op->getResultTypes(), isResourceType); | |
| } | |
| static void expandType(Type type, SmallVectorImpl<Type> &newTypes) { | |
| newTypes.push_back(type); | |
| if (isResourceType(type)) { | |
| // resource size, subrance offset, subrange length | |
| newTypes.append(3, IndexType::get(type.getContext())); | |
| } | |
| } | |
| // Expands resources in the given |types| list to (resource, size, offset, len). | |
| // This could be changed to some iterator magic to avoid the alloc. | |
| static SmallVector<Type> expandTypes(TypeRange types) { | |
| if (types.empty()) | |
| return {}; | |
| SmallVector<Type> newTypes; | |
| newTypes.reserve(types.size() * 2); | |
| for (auto type : types) { | |
| expandType(type, newTypes); | |
| } | |
| return newTypes; | |
| } | |
| struct Subrange { | |
| Value resource; | |
| Value resourceSize; | |
| Value subrangeOffset; | |
| Value subrangeLength; | |
| IREE::Util::SubrangeTypeInterface getResourceType() { | |
| return llvm::cast<IREE::Util::SubrangeTypeInterface>(resource.getType()); | |
| } | |
| }; | |
| using SubrangeMap = llvm::DenseMap<Value, Subrange>; | |
| // Attempts to find and consume a subrange associated with |value|. | |
| // Returns the subrange - which may point at a different resource than |value|. | |
| // In cases where no associated subrange was found the subrange will cover the | |
| // entire resource (offset at 0, length at size). | |
| static Subrange consumeSubrange(Location loc, Value value, | |
| SubrangeMap &subrangeMap, IndexSet &indexSet, | |
| OpBuilder &builder) { | |
| // TODO(benvanik): follow ties on value to try to consume there; there are a | |
| // few other ops we could look through as well (such as select, where we could | |
| // join). For now we just look at immediate defining ops. | |
| auto mapIt = subrangeMap.find(value); | |
| if (mapIt != subrangeMap.end()) { | |
| return mapIt->second; | |
| } | |
| if (auto subrangeOp = dyn_cast_or_null<IREE::Util::SubrangeOpInterface>( | |
| value.getDefiningOp())) { | |
| Subrange subrange; | |
| subrange.resource = subrangeOp.getSubrangeResource(); | |
| subrange.resourceSize = subrangeOp.getSubrangeResourceSize(); | |
| subrange.subrangeOffset = subrangeOp.getSubrangeOffset(); | |
| subrange.subrangeLength = subrangeOp.getSubrangeLength(); | |
| subrangeMap[value] = subrange; | |
| return subrange; | |
| } else { | |
| Subrange subrange; | |
| subrange.resource = value; | |
| subrange.resourceSize = | |
| IREE::Util::SizeAwareTypeInterface::queryValueSize(loc, value, builder); | |
| subrange.subrangeOffset = indexSet.get(0); | |
| subrange.subrangeLength = subrange.resourceSize; | |
| subrangeMap[value] = subrange; | |
| return subrange; | |
| } | |
| } | |
| static void expandOperand(Location loc, Value operand, | |
| SmallVectorImpl<Value> &newOperands, | |
| SubrangeMap &subrangeMap, IndexSet &indexSet, | |
| OpBuilder &builder) { | |
| if (isResourceType(operand.getType())) { | |
| auto subrange = | |
| consumeSubrange(loc, operand, subrangeMap, indexSet, builder); | |
| llvm::append_values(newOperands, subrange.resource, subrange.resourceSize, | |
| subrange.subrangeOffset, subrange.subrangeLength); | |
| } else { | |
| newOperands.push_back(operand); | |
| } | |
| } | |
| // Expands resources in |operands| into (resource, size, offset, length) tuples. | |
| static SmallVector<Value> expandOperands(Location loc, ValueRange operands, | |
| SubrangeMap &subrangeMap, | |
| IndexSet &indexSet, | |
| OpBuilder &builder) { | |
| SmallVector<Value> result; | |
| result.reserve(operands.size() * 2); | |
| for (auto operand : operands) { | |
| expandOperand(loc, operand, result, subrangeMap, indexSet, builder); | |
| } | |
| return result; | |
| } | |
| static void expandSubranges(Operation *op, SymbolTable &symbolTable, | |
| ExpandedGlobalMap &globalMap, IndexSet &indexSet, | |
| SubrangeMap &subrangeMap); | |
| // Recursively expands resources into (resource, size, offset, length) tuples | |
| // within the given |region|. All branches, ops, and nested regions will be | |
| // processed. | |
| static void expandRegion(Region ®ion, bool canModifyEntryBlock, | |
| SymbolTable &symbolTable, ExpandedGlobalMap &globalMap, | |
| IndexSet &indexSet, SubrangeMap subrangeMap) { | |
| if (region.empty()) | |
| return; | |
| // Update all block arguments. | |
| auto indexType = IndexType::get(region.getContext()); | |
| for (auto &block : region.getBlocks()) { | |
| if (!llvm::any_of(block.getArgumentTypes(), isResourceType)) | |
| continue; | |
| if (block.isEntryBlock() && !canModifyEntryBlock) | |
| continue; | |
| // Insert and build a list of expanded (resource, size, offset) tuples. | |
| SmallVector<Subrange> expansions; | |
| for (int i = block.getNumArguments() - 1; i >= 0; --i) { | |
| auto arg = block.getArgument(i); | |
| if (!isResourceType(arg.getType())) | |
| continue; | |
| Subrange subrange; | |
| subrange.resource = arg; | |
| subrange.resourceSize = | |
| block.insertArgument(i + 1, indexType, arg.getLoc()); | |
| subrange.subrangeOffset = | |
| block.insertArgument(i + 2, indexType, arg.getLoc()); | |
| subrange.subrangeLength = | |
| block.insertArgument(i + 3, indexType, arg.getLoc()); | |
| expansions.push_back(subrange); | |
| subrangeMap[arg] = subrange; | |
| } | |
| // Insert subranges that we've sunk from callers. | |
| auto builder = OpBuilder::atBlockBegin(&block); | |
| for (auto &expansion : llvm::reverse(expansions)) { | |
| auto newSubrange = expansion.getResourceType().createSubrangeOp( | |
| region.getLoc(), expansion.resource, expansion.resourceSize, | |
| expansion.subrangeOffset, expansion.subrangeLength, builder); | |
| expansion.resource.replaceAllUsesExcept(newSubrange, | |
| newSubrange.getDefiningOp()); | |
| } | |
| } | |
| // Walk blocks forward in domination order so that we add dominating values to | |
| // the offset map. Note that DominanceInfo is just determined not to be | |
| // cool about things when there's only one block so we have to special case. | |
| if (region.hasOneBlock()) { | |
| for (auto &op : | |
| llvm::make_early_inc_range(region.front().getOperations())) { | |
| expandSubranges(&op, symbolTable, globalMap, indexSet, subrangeMap); | |
| } | |
| } else { | |
| DominanceInfo domInfo(region.getParentOp()); | |
| for (auto *blockInfo : llvm::breadth_first(domInfo.getRootNode(®ion))) { | |
| auto *block = blockInfo->getBlock(); | |
| for (auto &op : llvm::make_early_inc_range(block->getOperations())) { | |
| expandSubranges(&op, symbolTable, globalMap, indexSet, subrangeMap); | |
| } | |
| } | |
| } | |
| } | |
| // Recursively expands all regions on the op. | |
| static void expandRegions(Operation *op, bool canModifyEntryBlock, | |
| SymbolTable &symbolTable, | |
| ExpandedGlobalMap &globalMap, IndexSet &indexSet, | |
| SubrangeMap subrangeMap) { | |
| for (auto ®ion : op->getRegions()) { | |
| expandRegion(region, canModifyEntryBlock, symbolTable, globalMap, indexSet, | |
| subrangeMap); | |
| } | |
| } | |
| // Updates the |subrangeMap| with the combined subrange of |op| and its source. | |
| static void updateSubrangeOp(IREE::Util::SubrangeOpInterface op, | |
| IndexSet &indexSet, SubrangeMap &subrangeMap) { | |
| // Ignore ops that are already in the map (we likely inserted them ourselves | |
| // earlier). | |
| auto resultResource = op.getSubrangeResult(); | |
| if (!resultResource) | |
| return; | |
| if (subrangeMap.count(resultResource)) | |
| return; | |
| // Get the subrange of the source resource which we should have by way of the | |
| // other insertions (func/block args, etc). | |
| OpBuilder builder(op); | |
| builder.setInsertionPointAfter(op); | |
| auto sourceSubrange = consumeSubrange(op.getLoc(), op.getSubrangeResource(), | |
| subrangeMap, indexSet, builder); | |
| if (op.getSubrangeResource() == sourceSubrange.resource) | |
| return; | |
| // Update the subrange in the map by adding the source offset and the local | |
| // offset from the op. Future ops that consume subranges will reference back | |
| // to the source with this subrange. | |
| Subrange updatedSubrange; | |
| updatedSubrange.resource = sourceSubrange.resource; | |
| updatedSubrange.resourceSize = sourceSubrange.resourceSize; | |
| updatedSubrange.subrangeOffset = builder.createOrFold<arith::AddIOp>( | |
| op.getLoc(), sourceSubrange.subrangeOffset, op.getSubrangeOffset()); | |
| updatedSubrange.subrangeLength = op.getSubrangeLength(); | |
| subrangeMap[resultResource] = updatedSubrange; | |
| } | |
| // Moves resource subranges from global stores to loads. | |
| // Requires that the ExpandGlobalStoreOp pattern elides the await. | |
| // | |
| // Example: | |
| // %0 = util.global.load @foo : !stream.resource | |
| // -> | |
| // %0 = util.global.load @foo : !stream.resource | |
| // %s = util.global.load @foo_size : index | |
| // %o = util.global.load @foo_offset : index | |
| // %l = util.global.load @foo_length : index | |
| // %1 = stream.resource.subview %0[%o] : | |
| // !stream.resource<*>{%s} -> !stream.resource<*>{%l} | |
| static void expandGlobalLoadOp(IREE::Util::GlobalLoadOpInterface op, | |
| ExpandedGlobalMap &globalMap, IndexSet &indexSet, | |
| SubrangeMap &subrangeMap) { | |
| if (!usesResources(op)) | |
| return; | |
| OpBuilder builder(op); | |
| builder.setInsertionPointAfter(op); | |
| auto &expandedGlobal = globalMap[op.getGlobalName()]; | |
| Subrange subrange; | |
| subrange.resource = op.getLoadedGlobalValue(); | |
| subrange.resourceSize = | |
| expandedGlobal.resourceSizeOp.createLoadOp(op.getLoc(), builder) | |
| .getLoadedGlobalValue(); | |
| subrange.subrangeOffset = | |
| expandedGlobal.subrangeOffsetOp.createLoadOp(op.getLoc(), builder) | |
| .getLoadedGlobalValue(); | |
| subrange.subrangeLength = | |
| expandedGlobal.subrangeLengthOp.createLoadOp(op.getLoc(), builder) | |
| .getLoadedGlobalValue(); | |
| subrangeMap[op.getLoadedGlobalValue()] = subrange; | |
| auto newSubrange = subrange.getResourceType().createSubrangeOp( | |
| op.getLoc(), subrange.resource, subrange.resourceSize, | |
| subrange.subrangeOffset, subrange.subrangeLength, builder); | |
| op.getLoadedGlobalValue().replaceAllUsesExcept(newSubrange, | |
| newSubrange.getDefiningOp()); | |
| } | |
| // Moves resource subranges from global stores to loads. | |
| // Requires that the ExpandGlobalLoadOp pattern inserts the await. | |
| // | |
| // Example: | |
| // %1 = stream.resource.subview %0[%o] : | |
| // !stream.resource<*>{%s} -> !stream.resource<*>{%l} | |
| // util.global.store %1, @foo : !stream.resource | |
| // -> | |
| // util.global.store %0, @foo : !stream.resource | |
| // util.global.store %s, @foo_size : index | |
| // util.global.store %o, @foo_offset : index | |
| // util.global.store %l, @foo_length : index | |
| static void expandGlobalStoreOp(IREE::Util::GlobalStoreOpInterface op, | |
| ExpandedGlobalMap &globalMap, | |
| IndexSet &indexSet, SubrangeMap &subrangeMap) { | |
| if (!usesResources(op)) | |
| return; | |
| OpBuilder builder(op); | |
| builder.setInsertionPointAfter(op); | |
| auto subrange = consumeSubrange(op.getLoc(), op.getStoredGlobalValue(), | |
| subrangeMap, indexSet, builder); | |
| auto &expandedGlobal = globalMap[op.getGlobalName()]; | |
| builder.create<IREE::Util::GlobalStoreOp>( | |
| op.getLoc(), subrange.resource, expandedGlobal.resourceOp.getName()); | |
| builder.create<IREE::Util::GlobalStoreOp>( | |
| op.getLoc(), subrange.resourceSize, | |
| expandedGlobal.resourceSizeOp.getName()); | |
| builder.create<IREE::Util::GlobalStoreOp>( | |
| op.getLoc(), subrange.subrangeOffset, | |
| expandedGlobal.subrangeOffsetOp.getName()); | |
| builder.create<IREE::Util::GlobalStoreOp>( | |
| op.getLoc(), subrange.subrangeLength, | |
| expandedGlobal.subrangeLengthOp.getName()); | |
| op.erase(); | |
| } | |
| static void expandInitializerOp(IREE::Util::InitializerOp op, | |
| SymbolTable &symbolTable, | |
| ExpandedGlobalMap &globalMap, | |
| IndexSet &indexSet, SubrangeMap &subrangeMap) { | |
| expandRegion(op.getRegion(), /*canModifyEntryBlock=*/false, symbolTable, | |
| globalMap, indexSet, subrangeMap); | |
| } | |
| // Inserts subranges on resource arguments. | |
| // Requires that the ExpandCallOp/ExpandReturnOp patterns handle migrating the | |
| // await. | |
| // | |
| // NOTE: this needs IPO to remove redundant subranges in cases where the call | |
| // sites don't need a wait. | |
| // | |
| // Example: | |
| // util.func @foo(%0: !stream.resource) | |
| // -> | |
| // util.func @foo(%0: !stream.resource, %sz: index, %o: index, %l: index) { | |
| // %1 = stream.resource.subview %0[%o] : {%sz} -> {%l} | |
| static void expandFuncOp(IREE::Util::FuncOp op, SymbolTable &symbolTable, | |
| ExpandedGlobalMap &globalMap, IndexSet &indexSet, | |
| SubrangeMap &subrangeMap) { | |
| // Ignore public/external function signatures but still convert regions. | |
| bool canModifyEntryBlock = !IREE::Util::isPublicOrExternal(op); | |
| if (canModifyEntryBlock) { | |
| op.expandSignature( | |
| [&](unsigned i, Type type, SmallVectorImpl<Type> &newTypes) { | |
| expandType(type, newTypes); | |
| }, | |
| [&](unsigned i, Type type, SmallVectorImpl<Type> &newTypes) { | |
| expandType(type, newTypes); | |
| }); | |
| } | |
| expandRegion(op.getRegion(), canModifyEntryBlock, symbolTable, globalMap, | |
| indexSet, subrangeMap); | |
| } | |
| // Splits resource operands and results into (resource, resourceSize, | |
| // subrangeOffset, subrangeLength). | |
| // Requires that the ExpandFuncOp/ExpandReturnOp patterns handle migrating the | |
| // await. | |
| // | |
| // NOTE: this needs IPO to remove redundant values in cases where the call sites | |
| // don't need a subrange. | |
| // | |
| // Example: | |
| // %1 = stream.resource.subview %0[%o] : {%sz} -> {%l} | |
| // %r = util.call @foo(%1) | |
| // -> | |
| // %r, %rsz, %ro, %rl = util.call @foo(%0, %sz, %o, %l) | |
| // %2 = stream.resource.subview %r[%ro] : {%rsz} -> {%rl} | |
| static void expandCallOp(IREE::Util::CallOp op, SymbolTable &symbolTable, | |
| IndexSet &indexSet, SubrangeMap &subrangeMap) { | |
| if (!usesResources(op)) | |
| return; | |
| // Ignore calls to public/external functions. | |
| auto calleeOp = symbolTable.lookup<CallableOpInterface>(op.getCallee()); | |
| if (IREE::Util::isPublicOrExternal(calleeOp)) | |
| return; | |
| // Build the new call op with expanded operands and results. | |
| OpBuilder builder(op); | |
| auto newOp = op.cloneAndExpand( | |
| [&](unsigned i, Value operand, SmallVectorImpl<Value> &newOperands) { | |
| expandOperand(op.getLoc(), operand, newOperands, subrangeMap, indexSet, | |
| builder); | |
| }, | |
| [&](unsigned i, Type type, SmallVectorImpl<Type> &newTypes) { | |
| expandType(type, newTypes); | |
| }, | |
| builder); | |
| // Insert subranges on results that we are sinking across the call edge. | |
| // The hope is that by moving the subranges here we can fold with uses inside | |
| // of this function. | |
| builder.setInsertionPointAfter(newOp); | |
| unsigned newIdx = 0; | |
| for (unsigned oldIdx = 0; oldIdx < op.getNumResults(); ++oldIdx) { | |
| auto oldResult = op.getResult(oldIdx); | |
| if (!isResourceType(oldResult.getType())) { | |
| auto newResult = newOp.getResult(newIdx++); | |
| oldResult.replaceAllUsesWith(newResult); | |
| continue; | |
| } | |
| Subrange subrange; | |
| subrange.resource = newOp.getResult(newIdx++); | |
| subrange.resourceSize = newOp.getResult(newIdx++); | |
| subrange.subrangeOffset = newOp.getResult(newIdx++); | |
| subrange.subrangeLength = newOp.getResult(newIdx++); | |
| subrangeMap[subrange.resource] = subrange; | |
| auto newSubrange = subrange.getResourceType().createSubrangeOp( | |
| op.getLoc(), subrange.resource, subrange.resourceSize, | |
| subrange.subrangeOffset, subrange.subrangeLength, builder); | |
| oldResult.replaceAllUsesWith(newSubrange); | |
| } | |
| op.erase(); | |
| } | |
| // Moves subranges to callers upon return. | |
| // Requires that the ExpandFuncOp/ExpandCallOp patterns handle migrating the | |
| // await. | |
| // | |
| // Example: | |
| // %1 = stream.resource.subview %0[%o] : {%sz} -> {%l} | |
| // util.return %1 | |
| // -> | |
| // util.return %0, %sz, %o, %l | |
| static void expandReturnOp(IREE::Util::ReturnOp op, IndexSet &indexSet, | |
| SubrangeMap &subrangeMap) { | |
| if (!usesResources(op)) | |
| return; | |
| if (IREE::Util::isPublicOrExternal(op->getParentOfType<IREE::Util::FuncOp>())) | |
| return; | |
| OpBuilder builder(op); | |
| auto operands = expandOperands(op.getLoc(), op.getOperands(), subrangeMap, | |
| indexSet, builder); | |
| builder.create<IREE::Util::ReturnOp>(op.getLoc(), operands); | |
| op.erase(); | |
| } | |
| // Moves subranges across branches. | |
| // Requires that the ExpandFuncOp pattern handles modifying the block args. | |
| // | |
| // Example: | |
| // %1 = stream.resource.subview %0[%o] : {%sz} -> {%l} | |
| // br ^bb1(%1) | |
| // ^bb1(%b): | |
| // -> | |
| // br ^bb1(%0, %sz, %o, %l) | |
| // ^bb1(%a, %b, %c, %d): | |
| // %1 = stream.resource.subview %a[%b] : {%c} -> {%d} | |
| static void expandBranchOp(mlir::cf::BranchOp op, IndexSet &indexSet, | |
| SubrangeMap &subrangeMap) { | |
| OpBuilder builder(op); | |
| auto operands = expandOperands(op.getLoc(), op.getDestOperands(), subrangeMap, | |
| indexSet, builder); | |
| builder.create<mlir::cf::BranchOp>(op.getLoc(), op.getDest(), operands); | |
| op.erase(); | |
| } | |
| static void expandCondBranchOp(mlir::cf::CondBranchOp op, IndexSet &indexSet, | |
| SubrangeMap &subrangeMap) { | |
| if (!usesResources(op)) | |
| return; | |
| OpBuilder builder(op); | |
| builder.create<mlir::cf::CondBranchOp>( | |
| op.getLoc(), op.getCondition(), op.getTrueDest(), | |
| expandOperands(op.getLoc(), op.getTrueDestOperands(), subrangeMap, | |
| indexSet, builder), | |
| op.getFalseDest(), | |
| expandOperands(op.getLoc(), op.getFalseDestOperands(), subrangeMap, | |
| indexSet, builder)); | |
| op.erase(); | |
| } | |
| static ValueRange asValueRange(ArrayRef<Value> values) { return values; } | |
| static void expandSwitchOp(mlir::cf::SwitchOp op, IndexSet &indexSet, | |
| SubrangeMap &subrangeMap) { | |
| if (!usesResources(op)) | |
| return; | |
| OpBuilder builder(op); | |
| auto caseOperands = llvm::to_vector( | |
| llvm::map_range(op.getCaseOperands(), [&](ValueRange operands) { | |
| return expandOperands(op.getLoc(), operands, subrangeMap, indexSet, | |
| builder); | |
| })); | |
| builder.create<mlir::cf::SwitchOp>( | |
| op.getLoc(), op.getFlag(), op.getDefaultDestination(), | |
| expandOperands(op.getLoc(), op.getDefaultOperands(), subrangeMap, | |
| indexSet, builder), | |
| op.getCaseValuesAttr(), op.getCaseDestinations(), | |
| llvm::to_vector(llvm::map_range(caseOperands, asValueRange))); | |
| op.erase(); | |
| } | |
| // Recursively expands resources into (resource, size, offset, length) in |op|. | |
| // TODO(benvanik): make this a type switch. | |
| static void expandSubranges(Operation *op, SymbolTable &symbolTable, | |
| ExpandedGlobalMap &globalMap, IndexSet &indexSet, | |
| SubrangeMap &subrangeMap) { | |
| if (auto subrangeOp = dyn_cast<IREE::Util::SubrangeOpInterface>(op)) { | |
| return updateSubrangeOp(subrangeOp, indexSet, subrangeMap); | |
| } | |
| if (auto loadOp = dyn_cast<IREE::Util::GlobalLoadOpInterface>(op)) { | |
| return expandGlobalLoadOp(loadOp, globalMap, indexSet, subrangeMap); | |
| } else if (auto storeOp = dyn_cast<IREE::Util::GlobalStoreOpInterface>(op)) { | |
| return expandGlobalStoreOp(storeOp, globalMap, indexSet, subrangeMap); | |
| } else if (auto initializerOp = dyn_cast<IREE::Util::InitializerOp>(op)) { | |
| return expandInitializerOp(initializerOp, symbolTable, globalMap, indexSet, | |
| subrangeMap); | |
| } else if (auto funcOp = dyn_cast<IREE::Util::FuncOp>(op)) { | |
| return expandFuncOp(funcOp, symbolTable, globalMap, indexSet, subrangeMap); | |
| } else if (auto callOp = dyn_cast<IREE::Util::CallOp>(op)) { | |
| return expandCallOp(callOp, symbolTable, indexSet, subrangeMap); | |
| } else if (auto returnOp = dyn_cast<IREE::Util::ReturnOp>(op)) { | |
| return expandReturnOp(returnOp, indexSet, subrangeMap); | |
| } else if (auto branchOp = dyn_cast<mlir::cf::BranchOp>(op)) { | |
| return expandBranchOp(branchOp, indexSet, subrangeMap); | |
| } else if (auto condBranchOp = dyn_cast<mlir::cf::CondBranchOp>(op)) { | |
| return expandCondBranchOp(condBranchOp, indexSet, subrangeMap); | |
| } else if (auto switchOp = dyn_cast<mlir::cf::SwitchOp>(op)) { | |
| return expandSwitchOp(switchOp, indexSet, subrangeMap); | |
| } | |
| // We could have a more generic thing here with RegionBranchOpInterface but | |
| // not all ops can contain subrange ops and some are isolated from above. | |
| // We could add an interface to ops we want to do this to, though, to at least | |
| // allow dialects to plug in. For now we just need SCF so this is hardcoded. | |
| if (auto ifOp = dyn_cast<mlir::scf::IfOp>(op)) { | |
| return expandRegions(ifOp, /*canModifyEntryBlock=*/false, symbolTable, | |
| globalMap, indexSet, subrangeMap); | |
| } else if (auto forOp = dyn_cast<mlir::scf::ForOp>(op)) { | |
| return expandRegions(forOp, /*canModifyEntryBlock=*/false, symbolTable, | |
| globalMap, indexSet, subrangeMap); | |
| } else if (auto whileOp = dyn_cast<mlir::scf::WhileOp>(op)) { | |
| return expandRegions(whileOp, /*canModifyEntryBlock=*/false, symbolTable, | |
| globalMap, indexSet, subrangeMap); | |
| } | |
| // TODO(benvanik): also handle scf.yield: today we don't propagate across | |
| // return values. | |
| } | |
| //===----------------------------------------------------------------------===// | |
| // -iree-util-propagate-subranges | |
| //===----------------------------------------------------------------------===// | |
| // This does a relatively mechanical transformation of a module to expand all | |
| // resource values (and globals) into (resource, size, offset, length) tuples. | |
| // | |
| // This is designed to be composed with generic optimization passes like global | |
| // fusion/folding and IPO and as such performs all transformations locally. For | |
| // example, calls are always updated to take/return subrange ranges and results | |
| // are always wrapped in a subrange op, with the elision/deduplication/etc left | |
| // until cleanup. | |
| class PropagateSubrangesPass | |
| : public impl::PropagateSubrangesPassBase<PropagateSubrangesPass> { | |
| public: | |
| void runOnOperation() override { | |
| auto rootOp = getOperation(); | |
| SymbolTable symbolTable(rootOp); | |
| // Expand all util.global ops holding resources into resource and subrange. | |
| auto globalMap = expandResourceGlobals(rootOp, symbolTable); | |
| // Walk the entire IR tree and expand the globals. | |
| // We could do this via pattern application but that gets much trickier to | |
| // manage with the expansion as we'd need to prevent ourselves from | |
| // expanding multiple times. | |
| for (auto callableOp : rootOp.getOps<mlir::CallableOpInterface>()) { | |
| // NOTE: the callable may be empty (like when an extern) - we still want | |
| // to process it but don't need an IndexSet. | |
| auto *region = callableOp.getCallableRegion(); | |
| if (!region || region->empty()) | |
| continue; | |
| IndexSet indexSet(callableOp.getLoc(), | |
| OpBuilder::atBlockBegin(®ion->front())); | |
| SubrangeMap subrangeMap; | |
| expandSubranges(callableOp, symbolTable, globalMap, indexSet, | |
| subrangeMap); | |
| } | |
| } | |
| }; | |
| } // namespace | |
| } // namespace mlir::iree_compiler::IREE::Util |