| // Copyright 2020 The IREE Authors |
| // |
| // Licensed under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| |
| #include <cstdlib> |
| #include <unordered_set> |
| |
| #include "compiler/plugins/target/LLVMCPU/Builtins/Device.h" |
| #include "compiler/plugins/target/LLVMCPU/Builtins/Musl.h" |
| #include "compiler/plugins/target/LLVMCPU/Builtins/UKernel.h" |
| #include "compiler/plugins/target/LLVMCPU/LLVMIRPasses.h" |
| #include "compiler/plugins/target/LLVMCPU/LLVMTargetOptions.h" |
| #include "compiler/plugins/target/LLVMCPU/LibraryBuilder.h" |
| #include "compiler/plugins/target/LLVMCPU/LinkerTool.h" |
| #include "compiler/plugins/target/LLVMCPU/StaticLibraryGenerator.h" |
| #include "iree/compiler/Codegen/Dialect/Codegen/IR/IREECodegenDialect.h" |
| #include "iree/compiler/Codegen/LLVMCPU/Passes.h" |
| #include "iree/compiler/Codegen/LLVMCPU/Utils.h" |
| #include "iree/compiler/Codegen/Utils/Utils.h" |
| #include "iree/compiler/Dialect/HAL/Target/Devices/LocalDevice.h" |
| #include "iree/compiler/Dialect/HAL/Target/TargetRegistry.h" |
| #include "iree/compiler/Dialect/HAL/Utils/LLVMLinkerUtils.h" |
| #include "iree/compiler/Dialect/LinalgExt/IR/LinalgExtDialect.h" |
| #include "iree/compiler/PluginAPI/Client.h" |
| #include "iree/compiler/Utils/ModuleUtils.h" |
| #include "llvm/Bitcode/BitcodeReader.h" |
| #include "llvm/Bitcode/BitcodeWriter.h" |
| #include "llvm/IR/GlobalValue.h" |
| #include "llvm/IR/LLVMContext.h" |
| #include "llvm/IR/Module.h" |
| #include "llvm/Linker/Linker.h" |
| #include "llvm/Support/TargetSelect.h" |
| #include "mlir/Dialect/ArmNeon/ArmNeonDialect.h" |
| #include "mlir/Dialect/ArmSME/IR/ArmSME.h" |
| #include "mlir/Dialect/ArmSVE/IR/ArmSVEDialect.h" |
| #include "mlir/Dialect/LLVMIR/LLVMDialect.h" |
| #include "mlir/Dialect/PDL/IR/PDL.h" |
| #include "mlir/Dialect/PDLInterp/IR/PDLInterp.h" |
| #include "mlir/Dialect/Transform/IR/TransformDialect.h" |
| #include "mlir/IR/BuiltinAttributes.h" |
| #include "mlir/IR/DialectResourceBlobManager.h" |
| #include "mlir/Target/LLVMIR/Dialect/ArmSME/ArmSMEToLLVMIRTranslation.h" |
| #include "mlir/Target/LLVMIR/Dialect/ArmSVE/ArmSVEToLLVMIRTranslation.h" |
| #include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h" |
| #include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h" |
| #include "mlir/Target/LLVMIR/Export.h" |
| |
| #define DEBUG_TYPE "iree-llvm-cpu-target" |
| using llvm::dbgs; |
| |
| namespace mlir::iree_compiler::IREE::HAL { |
| |
| static constexpr char kQueryFunctionName[] = |
| "iree_hal_executable_library_query"; |
| |
| static void dumpLLVMModuleToPath(StringRef path, StringRef baseName, |
| StringRef suffix, StringRef extPrefix, |
| llvm::Module &module) { |
| // Dump disassembly to path. |
| llvm::SmallVector<char> textData; |
| llvm::raw_svector_ostream textOstream(textData); |
| |
| module.print(textOstream, nullptr); |
| std::string textExtension = extPrefix.str() + ".ll"; |
| dumpDataToPath(path, baseName, suffix, textExtension, |
| StringRef(textData.data(), textData.size())); |
| |
| // Dump bitcode to path. |
| llvm::SmallVector<char> binaryData; |
| llvm::raw_svector_ostream binaryOstream(binaryData); |
| // Write the specified module to the specified output stream. |
| llvm::WriteBitcodeToFile(module, binaryOstream); |
| std::string binaryExtension = extPrefix.str() + ".bc"; |
| dumpDataToPath(path, baseName, suffix, binaryExtension, |
| StringRef(binaryData.data(), binaryData.size())); |
| } |
| |
| static void fixupVisibility(llvm::Module &module, |
| const SetVector<llvm::Function *> &preserveFuncs) { |
| for (auto &func : module) { |
| if (preserveFuncs.contains(&func) || func.getName() == "iree_dll_main") { |
| // Leave our library query function as public/external so that it is |
| // exported from shared objects and available for linking in static |
| // objects. |
| continue; |
| } else if (func.isDeclaration()) { |
| // Declarations must have their original visibility/linkage; they most |
| // often come from declared llvm builtin ops (llvm.memcpy/etc). |
| continue; |
| } |
| func.setDSOLocal(true); |
| func.setLinkage(llvm::GlobalValue::LinkageTypes::InternalLinkage); |
| } |
| for (auto &global : module.globals()) { |
| global.setDSOLocal(true); |
| global.setLinkage(llvm::GlobalValue::LinkageTypes::InternalLinkage); |
| } |
| } |
| |
| // Appends the |debugDatabase| to the end of |baseFile| and writes the footer |
| // so the runtime can find it. |
| static LogicalResult appendDebugDatabase(std::vector<int8_t> &baseFile, |
| Artifact &debugFileArtifact) { |
| auto debugFileOr = debugFileArtifact.read(); |
| if (!debugFileOr.has_value()) { |
| return failure(); |
| } |
| auto debugFile = std::move(debugFileOr).value(); |
| |
| // NOTE: we align the sizes so that the files all start at nice offsets. |
| auto baseFileSize = IREE::Util::align(baseFile.size(), 16); |
| auto debugFileSize = IREE::Util::align(debugFile.size(), 16); |
| |
| // Matches iree_hal_system_executable_footer_t. |
| struct Footer { |
| uint8_t magic[8]; // IREEDBG\0 |
| uint32_t version; |
| uint32_t flags; |
| uint64_t libraryOffset; |
| uint64_t librarySize; |
| uint64_t debugOffset; |
| uint64_t debugSize; |
| } footer = {{0}}; |
| std::memcpy(footer.magic, "IREEDBG\0", sizeof(footer.magic)); |
| footer.version = 0; |
| footer.librarySize = baseFile.size(); |
| footer.debugOffset = baseFileSize; |
| footer.debugSize = debugFile.size(); |
| |
| baseFile.resize(baseFileSize + debugFileSize + sizeof(footer)); |
| std::memcpy(baseFile.data() + baseFileSize, debugFile.data(), |
| debugFile.size()); |
| std::memcpy(baseFile.data() + baseFileSize + debugFileSize, &footer, |
| sizeof(footer)); |
| return success(); |
| } |
| |
| class LLVMCPUTargetBackend final : public TargetBackend { |
| public: |
| explicit LLVMCPUTargetBackend(LLVMTargetOptions options) |
| : defaultOptions_(std::move(options)) {} |
| |
| std::string getLegacyDefaultDeviceID() const override { return "llvm-cpu"; } |
| |
| void getDefaultExecutableTargets( |
| MLIRContext *context, StringRef deviceID, DictionaryAttr deviceConfigAttr, |
| SmallVectorImpl<IREE::HAL::ExecutableTargetAttr> &executableTargetAttrs) |
| const override { |
| executableTargetAttrs.push_back( |
| getExecutableTarget(context, defaultOptions_.target)); |
| } |
| |
| void getHostExecutableTargets(MLIRContext *context, StringRef deviceID, |
| DictionaryAttr deviceConfigAttr, |
| SmallVectorImpl<IREE::HAL::ExecutableTargetAttr> |
| &executableTargetAttrs) const override { |
| std::optional<LLVMTarget> maybeTarget = LLVMTarget::createForHost(); |
| if (maybeTarget) { |
| executableTargetAttrs.push_back( |
| getExecutableTarget(context, *maybeTarget)); |
| } |
| } |
| |
| IREE::HAL::ExecutableTargetAttr |
| getExecutableTarget(MLIRContext *context, const LLVMTarget &target) const { |
| // Add some configurations to the `hal.executable.target` attribute. |
| Builder b(context); |
| SmallVector<NamedAttribute> configItems; |
| target.storeToConfigAttrs(context, configItems); |
| |
| // Compute the format used at runtime to select the executable loader. |
| std::string format; |
| if (target.linkStatic) { |
| // Static libraries are just string references when serialized so we don't |
| // need to specify the target architecture. |
| format += "static"; |
| } else { |
| // Construct the [loader]-[format]-[arch] triple. |
| llvm::Triple targetTriple(target.getTriple()); |
| if (target.getLinkEmbedded()) { |
| // Using the IREE embedded ELF format/loader. |
| format += "embedded-elf-"; |
| } else { |
| // System-specific shared library format. |
| format += "system-"; |
| switch (targetTriple.getObjectFormat()) { |
| case llvm::Triple::ObjectFormatType::COFF: |
| format += "dll-"; |
| break; |
| case llvm::Triple::ObjectFormatType::ELF: |
| format += "elf-"; |
| break; |
| case llvm::Triple::ObjectFormatType::MachO: |
| format += "dylib-"; |
| break; |
| case llvm::Triple::ObjectFormatType::Wasm: |
| format += "wasm-"; |
| break; |
| default: |
| format += "unknown-"; |
| break; |
| } |
| } |
| format += getIreeArchNameForTargetTriple(targetTriple); |
| } |
| return b.getAttr<IREE::HAL::ExecutableTargetAttr>( |
| b.getStringAttr("llvm-cpu"), b.getStringAttr(format), |
| b.getDictionaryAttr(configItems)); |
| } |
| |
| void getDependentDialects(DialectRegistry ®istry) const override { |
| mlir::registerBuiltinDialectTranslation(registry); |
| mlir::registerLLVMDialectTranslation(registry); |
| mlir::registerArmSMEDialectTranslation(registry); |
| mlir::registerArmSVEDialectTranslation(registry); |
| // TODO: make inclusion of ArmNeon conditional? |
| // clang-format off |
| registry.insert<IREE::Codegen::IREECodegenDialect, |
| IREE::LinalgExt::IREELinalgExtDialect, |
| mlir::transform::TransformDialect, |
| pdl::PDLDialect, |
| pdl_interp::PDLInterpDialect, |
| arm_neon::ArmNeonDialect, |
| arm_sme::ArmSMEDialect, |
| arm_sve::ArmSVEDialect>(); |
| // clang-format on |
| } |
| |
| void |
| buildConfigurationPassPipeline(IREE::HAL::ExecutableTargetAttr targetAttr, |
| OpPassManager &passManager) override { |
| buildLLVMCPUCodegenConfigurationPassPipeline(passManager); |
| } |
| |
| void buildTranslationPassPipeline(IREE::HAL::ExecutableTargetAttr targetAttr, |
| OpPassManager &passManager) override { |
| bool enableAArch64SME = isAArch64(targetAttr) && hasSMEFeature(targetAttr); |
| buildLLVMCPUCodegenPassPipeline(passManager, enableAArch64SME); |
| } |
| |
| void buildLinkingPassPipeline(OpPassManager &passManager) override { |
| buildLLVMCPULinkingPassPipeline(passManager, "llvm-cpu"); |
| } |
| |
| // Gets the LLVM target from |variantOp|. |
| // This will differ from the default options specified by command line flags |
| // whenever multi-targeting. |
| // Returns none and emits on failure. |
| std::optional<LLVMTarget> |
| getVariantTarget(IREE::HAL::ExecutableVariantOp variantOp) { |
| auto configAttr = variantOp.getTarget().getConfiguration(); |
| return LLVMTarget::loadFromConfigAttr(variantOp.getLoc(), configAttr, |
| defaultOptions_.target); |
| } |
| |
| LogicalResult serializeExecutable(const SerializationOptions &options, |
| IREE::HAL::ExecutableVariantOp variantOp, |
| OpBuilder &executableBuilder) override { |
| // Perform the translation in a separate context to avoid any |
| // multi-threading issues. |
| llvm::LLVMContext context; |
| auto maybeTarget = getVariantTarget(variantOp); |
| if (!maybeTarget) |
| return failure(); |
| const LLVMTarget &target = *maybeTarget; |
| LLVM_DEBUG(dbgs() << "LLVM-CPU SerializeExecutable:\n" |
| << "-----------------------------\n"; |
| target.print(dbgs())); |
| |
| // For debugging effective options in live builds, uncomment the following. |
| // dbgs() << "LLVM-CPU "; |
| // target.print(dbgs()); |
| |
| // We name our files after the executable name so that they are easy to |
| // track both during compilation (logs/artifacts/etc), as outputs (final |
| // intermediate code/binary files), and at runtime (loaded |
| // libraries/symbols/etc). |
| auto libraryName = |
| variantOp->getParentOfType<IREE::HAL::ExecutableOp>().getName().str(); |
| |
| // Validate flags for output mode. |
| if (target.getLinkEmbedded() && target.linkStatic) { |
| return variantOp.emitError() |
| << "cannot embed ELF and produce static library simultaneously"; |
| } |
| |
| // Try to create the LLVM target machine interface for the variant target. |
| auto targetMachine = createTargetMachine(target); |
| if (!targetMachine) { |
| return mlir::emitError(variantOp.getLoc()) |
| << "failed to create target machine for target triple '" |
| << target.getTriple() << "'"; |
| } |
| |
| // Specialize the module to the target triple. |
| // The executable will have been cloned into other ExecutableVariantOps for |
| // other triples so it's fine to mutate in-place. |
| const llvm::Triple &targetTriple = targetMachine->getTargetTriple(); |
| variantOp.getInnerModule()->setAttr( |
| LLVM::LLVMDialect::getTargetTripleAttrName(), |
| executableBuilder.getStringAttr(targetTriple.str())); |
| |
| // At this moment we are leaving MLIR LLVM dialect land translating module |
| // into target independent LLVMIR. |
| auto llvmModule = mlir::translateModuleToLLVMIR(variantOp.getInnerModule(), |
| context, libraryName); |
| if (!llvmModule) { |
| return variantOp.emitError() << "failed to translate the MLIR LLVM " |
| "dialect to the native llvm::Module"; |
| } |
| |
| // Configure the functions in the module. This may override defaults set |
| // during the MLIR->LLVM conversion. |
| for (auto &func : *llvmModule) { |
| // Enable frame pointers to ensure that stack unwinding works, e.g. in |
| // Tracy. In principle this could also be achieved by enabling unwind |
| // tables, but we tried that and that didn't work in Tracy (which uses |
| // libbacktrace), while enabling frame pointers worked. |
| // https://github.com/iree-org/iree/issues/3957 |
| func.addFnAttr("frame-pointer", "all"); |
| |
| // -ffreestanding-like behavior. |
| func.addFnAttr("no-builtins"); |
| |
| // Our dispatches are all hot - that's kind of the point. |
| // This may favor more aggressive optimizations. |
| func.addFnAttr("hot"); |
| } |
| |
| // Build the IREE HAL executable library metadata. The runtime uses this to |
| // find the entry point functions and their information. |
| LibraryBuilder::Mode libraryBuilderMode = |
| target.debugSymbols ? LibraryBuilder::Mode::INCLUDE_REFLECTION_ATTRS |
| : LibraryBuilder::Mode::NONE; |
| LibraryBuilder libraryBuilder(llvmModule.get(), libraryBuilderMode, |
| LibraryBuilder::Version::LATEST); |
| |
| switch (target.sanitizerKind) { |
| case SanitizerKind::kNone: { |
| libraryBuilder.setSanitizerKind(LibraryBuilder::SanitizerKind::NONE); |
| break; |
| } |
| case SanitizerKind::kAddress: { |
| libraryBuilder.setSanitizerKind(LibraryBuilder::SanitizerKind::ADDRESS); |
| for (auto &function : llvmModule->getFunctionList()) { |
| function.addFnAttr(llvm::Attribute::SanitizeAddress); |
| } |
| } break; |
| case SanitizerKind::kThread: { |
| libraryBuilder.setSanitizerKind(LibraryBuilder::SanitizerKind::THREAD); |
| for (auto &function : llvmModule->getFunctionList()) { |
| function.addFnAttr(llvm::Attribute::SanitizeThread); |
| } |
| } break; |
| } |
| |
| // Declare dynamically imported functions. |
| auto importsAttrName = |
| StringAttr::get(variantOp.getContext(), "hal.executable.imports"); |
| if (auto importsAttr = |
| variantOp->getAttrOfType<ArrayAttr>(importsAttrName)) { |
| for (auto importAttr : importsAttr.getAsValueRange<ArrayAttr>()) { |
| auto nameAttr = llvm::cast<StringAttr>(importAttr[0]); |
| auto weakAttr = llvm::cast<BoolAttr>(importAttr[1]); |
| libraryBuilder.addImport(nameAttr.getValue(), weakAttr.getValue()); |
| } |
| variantOp->removeAttr(importsAttrName); |
| } |
| |
| // Declare exported entry points. |
| auto align16 = llvm::Attribute::getWithAlignment(context, llvm::Align(16)); |
| for (auto exportOp : variantOp.getBlock().getOps<ExecutableExportOp>()) { |
| // Find the matching function in the LLVM module. |
| auto *llvmFunc = llvmModule->getFunction(exportOp.getName()); |
| if (!llvmFunc) |
| continue; |
| llvmFunc->setLinkage(llvm::GlobalValue::LinkageTypes::InternalLinkage); |
| llvmFunc->setDSOLocal(true); |
| |
| // Tag the function parameters in case they got removed during conversion. |
| // (%arg0: environment, %arg1: dispatch_state, %arg2: workgroup_state) |
| for (unsigned i = 0; i <= 2; ++i) { |
| llvmFunc->addParamAttr(i, llvm::Attribute::NonNull); |
| llvmFunc->addParamAttr(i, llvm::Attribute::NoAlias); |
| llvmFunc->addParamAttr(i, align16); |
| } |
| |
| LibraryBuilder::DispatchAttrs dispatchAttrs = {0}; |
| |
| // Entry points may optionally specify that they require workgroup local |
| // memory. We fetch that value here and plumb it through so the runtime |
| // knows how much memory to reserve and pass in. |
| dispatchAttrs.localMemorySize = exportOp.getWorkgroupLocalMemory() |
| .value_or(APInt(64, 0)) |
| .getSExtValue(); |
| |
| // Specify the constant and binding information used to validate |
| // dispatches. |
| if (auto layoutAttr = exportOp.getLayout()) { |
| dispatchAttrs.constantCount = layoutAttr.getConstants(); |
| dispatchAttrs.bindingCount = layoutAttr.getBindings().size(); |
| } |
| |
| LibraryBuilder::SourceLocation sourceLocation; |
| if (options.debugLevel >= 1) { |
| if (auto loc = findFirstFileLoc(exportOp.getLoc())) { |
| sourceLocation = {"", loc->getFilename().str(), loc->getLine()}; |
| } |
| } |
| SmallVector<LibraryBuilder::SourceLocation> stageLocations; |
| if (options.debugLevel >= 3) { |
| if (auto locsAttr = exportOp.getSourceLocsAttr()) { |
| for (auto locAttr : locsAttr.getValue()) { |
| if (auto loc = |
| findFirstFileLoc(cast<LocationAttr>(locAttr.getValue()))) { |
| stageLocations.push_back({ |
| locAttr.getName().str(), |
| loc->getFilename().str(), |
| loc->getLine(), |
| }); |
| } |
| } |
| } |
| } |
| libraryBuilder.addExport(exportOp.getName(), std::move(sourceLocation), |
| std::move(stageLocations), /*tag=*/"", |
| dispatchAttrs, llvmFunc); |
| } |
| |
| // Embed source files (if present). |
| if (auto sourcesAttr = variantOp.getSourcesAttr()) { |
| for (auto sourceAttr : sourcesAttr.getValue()) { |
| if (auto resourceAttr = dyn_cast_if_present<DenseResourceElementsAttr>( |
| sourceAttr.getValue())) { |
| auto handle = resourceAttr.getRawHandle(); |
| SmallVector<char> rawData; |
| llvm::append_range(rawData, handle.getBlob()->getData()); |
| libraryBuilder.addSourceFile(sourceAttr.getName(), |
| std::move(rawData)); |
| } |
| } |
| } |
| |
| auto queryFunctionName = std::string(kQueryFunctionName); |
| if (target.linkStatic) { |
| // Static library query functions must be unique to support multiple |
| // libraries in the same namespace. |
| queryFunctionName = libraryName + "_library_query"; |
| } |
| auto *queryLibraryFunc = libraryBuilder.build(queryFunctionName); |
| |
| // The query function must be exported for dynamic libraries. |
| queryLibraryFunc->setDSOLocal(false); |
| queryLibraryFunc->setVisibility( |
| llvm::GlobalValue::VisibilityTypes::DefaultVisibility); |
| queryLibraryFunc->setLinkage( |
| llvm::GlobalValue::LinkageTypes::ExternalLinkage); |
| queryLibraryFunc->setDLLStorageClass( |
| llvm::GlobalValue::DLLStorageClassTypes::DLLExportStorageClass); |
| |
| // If linking dynamically, find a suitable linker tool and configure the |
| // module with any options that tool requires. |
| std::unique_ptr<LinkerTool> linkerTool; |
| if (!target.linkStatic) { |
| // Grab a linker tool based on the options (and target environment). |
| // This uses the defaultOptions_ in order to get paths and such, which |
| // are environmental, but replace the target with the actual one. |
| LLVMTargetOptions options = defaultOptions_; |
| options.target = target; |
| linkerTool = LinkerTool::getForTarget(targetTriple, options); |
| if (!linkerTool) { |
| return mlir::emitError(variantOp.getLoc()) |
| << "failed to find a target linker for the given target triple '" |
| << targetTriple.str() << "'"; |
| } |
| |
| // Configure the module with any code generation options required later by |
| // linking (such as initializer functions). |
| if (failed(linkerTool->configureModule(llvmModule.get(), |
| {queryLibraryFunc}))) { |
| return variantOp.emitError() |
| << "failed to configure LLVM module for target linker"; |
| } |
| } |
| |
| // Specialize the module to our target machine. |
| llvmModule->setDataLayout(targetMachine->createDataLayout()); |
| llvmModule->setTargetTriple(targetMachine->getTargetTriple().str()); |
| |
| // Dump just the codegen bitcode before linking and optimization. |
| if (!options.dumpIntermediatesPath.empty()) { |
| dumpLLVMModuleToPath(options.dumpIntermediatesPath, options.dumpBaseName, |
| variantOp.getName(), ".codegen", *llvmModule); |
| } |
| |
| // Statically link libraries into our module prior to LLVM optimizations. |
| // This approximates LTO. |
| llvm::Linker moduleLinker(*llvmModule); |
| |
| // Link any bitcode files specified on the command line. |
| if (failed(linkCmdlineBitcodeFiles(variantOp.getLoc(), moduleLinker, |
| llvm::Linker::OverrideFromSrc, |
| *targetMachine, context))) { |
| return failure(); |
| } |
| |
| // Link any bitcode objects specified in executable.object attributes and |
| // specialize them for the current config. |
| if (failed(linkBitcodeObjects(variantOp.getLoc(), moduleLinker, |
| llvm::Linker::LinkOnlyNeeded, *targetMachine, |
| variantOp.getObjectsAttr(), context))) { |
| return failure(); |
| } |
| |
| // Link our libdevice after all codegen and user objects as they may |
| // reference it. Some of the functions in here are only known used after |
| // we perform LLVM ISel and need to be pulled in whether they are used or |
| // not. |
| if (failed(linkBitcodeModule( |
| variantOp.getLoc(), moduleLinker, llvm::Linker::OverrideFromSrc, |
| *targetMachine, "libdevice", |
| loadDeviceBitcode(targetMachine.get(), context), |
| [&](llvm::Module &module) { |
| specializeDeviceModule(variantOp, module, *targetMachine); |
| }))) { |
| return mlir::emitError(variantOp.getLoc()) |
| << "failed linking in builtin library for target triple '" |
| << targetTriple.str() << "'"; |
| } |
| |
| if (target.getLinkEmbedded()) { |
| // Link musl last and pull in all of it - this is sad but LLVM will take |
| // IR intrinsics and generate calls out to libc during code generation and |
| // we have no control over that - if we don't provide the symbols here |
| // then linking with ld will fail. |
| if (failed(linkBitcodeModule( |
| variantOp.getLoc(), moduleLinker, llvm::Linker::OverrideFromSrc, |
| *targetMachine, "libmusl", |
| loadMuslBitcode(targetMachine.get(), context)))) { |
| return mlir::emitError(variantOp.getLoc()) |
| << "failed linking in builtin library for target triple '" |
| << targetTriple.str() << "'"; |
| } |
| } |
| |
| if (target.linkUkernelBitcode) { |
| // Link in ukernel bitcode. |
| if (hasUkernel(variantOp.getTarget())) { |
| llvm::Expected<std::unique_ptr<llvm::Module>> bitcode = |
| loadUKernelBitcode(targetMachine.get(), context); |
| if (!bitcode) { |
| return mlir::emitError(variantOp.getLoc()) |
| << "failed to load ukernel bitcode: " |
| << llvm::toString(bitcode.takeError()); |
| } |
| |
| if (bitcode.get()) { |
| StringRef bitcodeName = bitcode.get()->getName(); |
| if (failed(linkBitcodeModule(variantOp.getLoc(), moduleLinker, |
| llvm::Linker::LinkOnlyNeeded, |
| *targetMachine, bitcodeName, |
| std::move(bitcode), {}))) { |
| return mlir::emitError(variantOp.getLoc()) |
| << "failed linking in architecture-specific ukernel bitcode " |
| "for target triple '" |
| << targetTriple.str() << "'"; |
| } |
| } |
| } |
| } |
| |
| // Strip any compiler identifiers that may have snuck in. We let the linker |
| // tag the module. |
| auto *llvmIdent = llvmModule->getNamedMetadata("llvm.ident"); |
| if (llvmIdent) |
| llvmIdent->clearOperands(); |
| |
| // Dump all linked bitcode prior to optimization. |
| if (!options.dumpIntermediatesPath.empty()) { |
| dumpLLVMModuleToPath(options.dumpIntermediatesPath, options.dumpBaseName, |
| variantOp.getName(), ".linked", *llvmModule); |
| } |
| |
| // LLVM opt passes that perform code generation optimizations/transformation |
| // similar to what a frontend would do. |
| if (failed( |
| runLLVMIRPasses(target, targetMachine.get(), llvmModule.get()))) { |
| return variantOp.emitError() |
| << "failed to run LLVM-IR opt passes for IREE::HAL::ExecutableOp " |
| "targeting '" |
| << targetTriple.str() << "'"; |
| } |
| |
| // Fixup visibility from any symbols we may link in - we want to hide all |
| // but the query entry point. |
| // Note: can't move this before runLLVMIRPasses at the moment, as further |
| // symbol references may still be created past this point, namely to math |
| // functions, e.g. `llvm.frem` lowering to a call to `fmodf`. |
| SetVector<llvm::Function *> preservedFuncs; |
| preservedFuncs.insert(queryLibraryFunc); |
| fixupVisibility(*llvmModule, preservedFuncs); |
| |
| // Dump bitcode post-linking and optimization. |
| if (!options.dumpIntermediatesPath.empty()) { |
| dumpLLVMModuleToPath(options.dumpIntermediatesPath, options.dumpBaseName, |
| variantOp.getName(), ".optimized", *llvmModule); |
| } |
| |
| SmallVector<Artifact> objectFiles; |
| |
| // Emit the base object file containing the bulk of our code. |
| // This must come first such that we have the proper library linking order. |
| { |
| // NOTE: today we just use a single object file, however if we wanted to |
| // scale code generation and linking we'd want to generate one per |
| // function (or something like that). A single object file is also |
| // instrumental to static library generation (which only supports one |
| // object file per library). |
| std::string objectData; |
| if (failed(runEmitObjFilePasses(targetMachine.get(), llvmModule.get(), |
| llvm::CodeGenFileType::ObjectFile, |
| &objectData))) { |
| return variantOp.emitError() |
| << "failed to compile LLVM-IR module to an object file"; |
| } |
| if (!options.dumpIntermediatesPath.empty()) { |
| dumpDataToPath(options.dumpIntermediatesPath, options.dumpBaseName, |
| variantOp.getName(), ".o", objectData); |
| } |
| auto objectFile = Artifact::createTemporary(libraryName, "o"); |
| auto &os = objectFile.outputFile->os(); |
| os << objectData; |
| os.flush(); |
| os.close(); |
| objectFiles.push_back(std::move(objectFile)); |
| } |
| |
| // Dump assembly listing after optimization, which is just a textual |
| // representation of the object file we generate below. |
| if (!options.dumpIntermediatesPath.empty()) { |
| std::string asmData; |
| if (failed(runEmitObjFilePasses(targetMachine.get(), llvmModule.get(), |
| llvm::CodeGenFileType::AssemblyFile, |
| &asmData))) { |
| return variantOp.emitError() |
| << "failed to compile LLVM-IR module to an assembly file"; |
| } |
| dumpDataToPath(options.dumpIntermediatesPath, options.dumpBaseName, |
| variantOp.getName(), ".s", asmData); |
| } |
| |
| // If custom object files were specified then add those to our artifact set. |
| // These will either be combined into the resulting static library or linked |
| // statically into the resulting dynamic library. |
| SmallVector<IREE::HAL::ExecutableObjectAttr> linkerObjectAttrs; |
| IREE::HAL::ExecutableObjectAttr::filterObjects(variantOp.getObjectsAttr(), |
| {".o", ".obj", ".a", ".lib"}, |
| linkerObjectAttrs); |
| for (auto [index, attr] : llvm::enumerate(linkerObjectAttrs)) { |
| auto objectAttr = llvm::cast<IREE::HAL::ExecutableObjectAttr>(attr); |
| if (auto dataAttr = objectAttr.getData()) { |
| objectFiles.push_back(Artifact::createTemporary( |
| objectFiles.front().path + "_object_" + std::to_string(index), |
| llvm::sys::path::extension(objectAttr.getPath()))); |
| } else { |
| auto absolutePath = objectAttr.getAbsolutePath(); |
| if (failed(absolutePath)) { |
| llvm::errs() |
| << "ERROR: referenced object file not found on any path; use " |
| "--iree-hal-executable-object-search-path= to add search " |
| "paths: " |
| << objectAttr << "\n"; |
| return failure(); |
| } |
| objectFiles.push_back(Artifact::fromFile(*absolutePath)); |
| } |
| } |
| |
| if (target.linkStatic) { |
| return serializeStaticLibraryExecutable(options, target, variantOp, |
| executableBuilder, libraryName, |
| queryFunctionName, objectFiles); |
| } else { |
| return serializeDynamicLibraryExecutable( |
| options, target, variantOp, executableBuilder, libraryName, |
| targetTriple, objectFiles, linkerTool.get()); |
| } |
| } |
| |
| LogicalResult serializeStaticLibraryExecutable( |
| const SerializationOptions &options, const LLVMTarget &target, |
| IREE::HAL::ExecutableVariantOp variantOp, OpBuilder &executableBuilder, |
| const std::string &libraryName, const std::string &queryFunctionName, |
| const SmallVector<Artifact> &objectFiles) { |
| if (objectFiles.size() != 1) { |
| // Static library output only supports single object libraries. |
| return variantOp.emitError() << "generating static libraries from " |
| "multiple object files is not supported"; |
| } |
| |
| // Copy the static object file to the specified output along with |
| // generated header file. |
| if (!outputStaticLibrary(libraryName, queryFunctionName, |
| target.staticLibraryOutput, objectFiles[0].path)) { |
| return variantOp.emitError() << "static library generation failed"; |
| } |
| |
| // Embed the library name in the executable binary op. This informs the |
| // loader which static library to load for the target binary. |
| std::vector<uint8_t> libraryNameVector(libraryName.begin(), |
| libraryName.end()); |
| executableBuilder.create<IREE::HAL::ExecutableBinaryOp>( |
| variantOp.getLoc(), variantOp.getSymName(), "static", |
| libraryNameVector); |
| |
| return success(); |
| } |
| |
| LogicalResult serializeDynamicLibraryExecutable( |
| const SerializationOptions &options, const LLVMTarget &target, |
| IREE::HAL::ExecutableVariantOp variantOp, OpBuilder &executableBuilder, |
| const std::string &libraryName, const llvm::Triple &targetTriple, |
| const SmallVector<Artifact> &objectFiles, LinkerTool *linkerTool) { |
| // Link the generated object files into a dylib. |
| auto linkArtifactsOr = |
| linkerTool->linkDynamicLibrary(libraryName, objectFiles); |
| if (!linkArtifactsOr.has_value()) { |
| return mlir::emitError(variantOp.getLoc()) |
| << "failed to link executable and generate target dylib (check " |
| "above for more specific error messages)"; |
| } |
| auto &linkArtifacts = linkArtifactsOr.value(); |
| if (defaultOptions_.keepLinkerArtifacts) { |
| mlir::emitRemark(variantOp.getLoc()) |
| << "linker artifacts for " << variantOp.getName() << " preserved:\n" |
| << " " << linkArtifacts.libraryFile.path; |
| linkArtifacts.keepAllFiles(); |
| for (auto &objectFile : objectFiles) { |
| objectFile.keep(); |
| } |
| } |
| |
| if (target.getLinkEmbedded()) { |
| // Load the linked ELF file and pack into an attr. |
| auto elfFile = linkArtifacts.libraryFile.read(); |
| if (!elfFile.has_value()) { |
| return variantOp.emitError() |
| << "failed to read back dylib temp file at " |
| << linkArtifacts.libraryFile.path; |
| } |
| if (!options.dumpBinariesPath.empty()) { |
| dumpDataToPath<int8_t>(options.dumpBinariesPath, options.dumpBaseName, |
| variantOp.getName(), ".so", *elfFile); |
| } |
| auto bufferAttr = DenseIntElementsAttr::get( |
| VectorType::get({static_cast<int64_t>(elfFile->size())}, |
| IntegerType::get(executableBuilder.getContext(), 8)), |
| std::move(elfFile.value())); |
| |
| // Add the binary to the parent hal.executable. |
| auto binaryOp = executableBuilder.create<IREE::HAL::ExecutableBinaryOp>( |
| variantOp.getLoc(), variantOp.getSymName(), |
| variantOp.getTarget().getFormat(), bufferAttr); |
| binaryOp.setMimeTypeAttr( |
| executableBuilder.getStringAttr("application/x-elf")); |
| } else { |
| const char *mimeType = nullptr; |
| const char *extension = ""; |
| switch (targetTriple.getObjectFormat()) { |
| case llvm::Triple::ObjectFormatType::COFF: |
| mimeType = "application/x-msdownload"; |
| extension = ".dll"; |
| break; |
| case llvm::Triple::ObjectFormatType::ELF: |
| mimeType = "application/x-elf"; |
| extension = ".so"; |
| break; |
| case llvm::Triple::ObjectFormatType::MachO: |
| mimeType = "application/x-dylib"; |
| extension = ".dylib"; |
| break; |
| case llvm::Triple::ObjectFormatType::Wasm: |
| mimeType = "application/wasm"; |
| extension = ".wasm"; |
| break; |
| default: |
| mimeType = "application/octet-stream"; |
| break; |
| } |
| |
| // Load the linked system library and optionally tag on the debug |
| // database. This debug database sits at the tail of the file and is |
| // ignored by system loaders and tools but still accessible to the runtime |
| // loader. Not all platforms have separate debug databases and need this. |
| auto libraryFileOr = linkArtifacts.libraryFile.read(); |
| if (!libraryFileOr.has_value()) { |
| return variantOp.emitError() |
| << "failed to read back dylib temp file at " |
| << linkArtifacts.libraryFile.path; |
| } |
| auto libraryFile = std::move(libraryFileOr).value(); |
| if (target.debugSymbols && linkArtifacts.debugFile.outputFile) { |
| if (failed(appendDebugDatabase(libraryFile, linkArtifacts.debugFile))) { |
| return variantOp.emitError() |
| << "failed to append debug database to dylib file"; |
| } |
| } |
| if (!options.dumpBinariesPath.empty()) { |
| dumpDataToPath<int8_t>(options.dumpBinariesPath, options.dumpBaseName, |
| variantOp.getName(), extension, libraryFile); |
| } |
| auto bufferAttr = DenseIntElementsAttr::get( |
| VectorType::get({static_cast<int64_t>(libraryFile.size())}, |
| IntegerType::get(executableBuilder.getContext(), 8)), |
| std::move(libraryFile)); |
| |
| // Add the binary to the parent hal.executable. |
| auto binaryOp = executableBuilder.create<IREE::HAL::ExecutableBinaryOp>( |
| variantOp.getLoc(), variantOp.getSymName(), |
| variantOp.getTarget().getFormat(), bufferAttr); |
| binaryOp.setMimeTypeAttr(executableBuilder.getStringAttr(mimeType)); |
| } |
| |
| return success(); |
| } |
| |
| private: |
| // Default options as registered from the command line. Should not be |
| // relied on outside of getDefaultDeviceTarget() since it represents |
| // a static "cross compiling" config and would override more specific |
| // settings. |
| const LLVMTargetOptions defaultOptions_; |
| }; |
| |
| struct LLVMCPUSession |
| : public PluginSession<LLVMCPUSession, LLVMCPUTargetCLOptions, |
| PluginActivationPolicy::DefaultActivated> { |
| void populateHALTargetDevices(IREE::HAL::TargetDeviceList &targets) { |
| // TODO(multi-device): move local device registration out. |
| // This exists here for backwards compat with the old |
| // iree-hal-target-backends flag that needs to look up the device by backend |
| // name. |
| // #hal.device.target<"llvm-cpu", ... |
| targets.add("llvm-cpu", [=]() { |
| LocalDevice::Options localDeviceOptions; |
| localDeviceOptions.defaultTargetBackends.push_back("llvm-cpu"); |
| localDeviceOptions.defaultHostBackends.push_back("llvm-cpu"); |
| return std::make_shared<LocalDevice>(localDeviceOptions); |
| }); |
| } |
| void populateHALTargetBackends(IREE::HAL::TargetBackendList &targets) { |
| // #hal.executable.target<"llvm-cpu", ... |
| targets.add("llvm-cpu", [=]() { |
| return std::make_shared<LLVMCPUTargetBackend>(options.getTargetOptions()); |
| }); |
| } |
| }; |
| |
| } // namespace mlir::iree_compiler::IREE::HAL |
| |
| IREE_DEFINE_COMPILER_OPTION_FLAGS( |
| mlir::iree_compiler::IREE::HAL::LLVMCPUTargetCLOptions); |
| |
| extern "C" bool iree_register_compiler_plugin_hal_target_llvm_cpu( |
| mlir::iree_compiler::PluginRegistrar *registrar) { |
| registrar->registerPlugin<mlir::iree_compiler::IREE::HAL::LLVMCPUSession>( |
| "hal_target_llvm_cpu"); |
| return true; |
| } |
