Merge pull request #6786 from MaheshRavishankar:main-to-google PiperOrigin-RevId: 391350379
diff --git a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/BUILD b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/BUILD similarity index 87% rename from iree/compiler/Dialect/HAL/Conversion/IREEToHAL/BUILD rename to iree/compiler/Dialect/HAL/Conversion/UtilToHAL/BUILD index 1587a93..568b9bb 100644 --- a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/BUILD +++ b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/BUILD
@@ -11,12 +11,12 @@ ) cc_library( - name = "IREEToHAL", + name = "UtilToHAL", srcs = [ - "ConvertIREEToHAL.cpp", + "ConvertUtilToHAL.cpp", ], hdrs = [ - "ConvertIREEToHAL.h", + "ConvertUtilToHAL.h", ], deps = [ "//iree/compiler/Dialect/HAL/IR",
diff --git a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/CMakeLists.txt b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/CMakeLists.txt similarity index 87% rename from iree/compiler/Dialect/HAL/Conversion/IREEToHAL/CMakeLists.txt rename to iree/compiler/Dialect/HAL/Conversion/UtilToHAL/CMakeLists.txt index eaa63ea..2e6cf64 100644 --- a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/CMakeLists.txt +++ b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/CMakeLists.txt
@@ -1,6 +1,6 @@ ################################################################################ # Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from # -# iree/compiler/Dialect/HAL/Conversion/IREEToHAL/BUILD # +# iree/compiler/Dialect/HAL/Conversion/UtilToHAL/BUILD # # # # Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary # # CMake-only content. # @@ -12,11 +12,11 @@ iree_cc_library( NAME - IREEToHAL + UtilToHAL HDRS - "ConvertIREEToHAL.h" + "ConvertUtilToHAL.h" SRCS - "ConvertIREEToHAL.cpp" + "ConvertUtilToHAL.cpp" DEPS MLIRIR MLIRStandard
diff --git a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/ConvertIREEToHAL.cpp b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/ConvertUtilToHAL.cpp similarity index 60% rename from iree/compiler/Dialect/HAL/Conversion/IREEToHAL/ConvertIREEToHAL.cpp rename to iree/compiler/Dialect/HAL/Conversion/UtilToHAL/ConvertUtilToHAL.cpp index c50c619..908520c 100644 --- a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/ConvertIREEToHAL.cpp +++ b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/ConvertUtilToHAL.cpp
@@ -4,7 +4,7 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/compiler/Dialect/HAL/Conversion/IREEToHAL/ConvertIREEToHAL.h" +#include "iree/compiler/Dialect/HAL/Conversion/UtilToHAL/ConvertUtilToHAL.h" #include "iree/compiler/Dialect/HAL/IR/HALOps.h" #include "iree/compiler/Dialect/Util/IR/UtilOps.h" @@ -75,64 +75,13 @@ } }; -template <typename T> -class GenericConvertTypesConversion : public OpConversionPattern<T> { - public: - using OpConversionPattern<T>::OpConversionPattern; - - LogicalResult matchAndRewrite( - T op, llvm::ArrayRef<Value> newOperands, - ConversionPatternRewriter &rewriter) const override { - SmallVector<Type> newTypes; - bool anyChanged = false; - for (auto oldNew : llvm::zip(op->getOperands(), newOperands)) { - auto oldValue = std::get<0>(oldNew); - auto newValue = std::get<1>(oldNew); - if (oldValue.getType() != newValue.getType()) { - anyChanged = true; - break; - } - } - for (auto oldType : op.getOperation()->getResultTypes()) { - auto newType = this->getTypeConverter()->convertType(oldType); - if (oldType != newType) anyChanged = true; - newTypes.push_back(newType); - } - if (!anyChanged) return failure(); - rewriter.replaceOpWithNewOp<T>(op, newTypes, newOperands, op->getAttrs()); - return success(); - } -}; - } // namespace -void populateIREEToHALPatterns(MLIRContext *context, ConversionTarget &target, +void populateUtilToHALPatterns(MLIRContext *context, ConversionTarget &target, TypeConverter &typeConverter, OwningRewritePatternList &patterns) { target.addIllegalOp<IREE::Util::DynamicShapeConstantOp>(); patterns.insert<DynamicShapeConstantOpConversion>(context); - - typeConverter.addConversion([&](IREE::Util::ListType type) { - auto elementType = typeConverter.convertType(type.getElementType()); - return IREE::Util::ListType::get(elementType); - }); - - target.addDynamicallyLegalOp<IREE::Util::ListCreateOp>( - [&](IREE::Util::ListCreateOp op) { - return typeConverter.isLegal(op.getType()); - }); - target.addDynamicallyLegalOp<IREE::Util::ListGetOp>( - [&](IREE::Util::ListGetOp op) { - return typeConverter.isLegal(op.getType()); - }); - target.addDynamicallyLegalOp<IREE::Util::ListSetOp>( - [&](IREE::Util::ListSetOp op) { - return typeConverter.isLegal(op.value().getType()); - }); - patterns.insert<GenericConvertTypesConversion<IREE::Util::ListCreateOp>, - GenericConvertTypesConversion<IREE::Util::ListGetOp>, - GenericConvertTypesConversion<IREE::Util::ListSetOp>>( - typeConverter, context); } } // namespace iree_compiler
diff --git a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/ConvertIREEToHAL.h b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/ConvertUtilToHAL.h similarity index 75% rename from iree/compiler/Dialect/HAL/Conversion/IREEToHAL/ConvertIREEToHAL.h rename to iree/compiler/Dialect/HAL/Conversion/UtilToHAL/ConvertUtilToHAL.h index 128110d..d6b7bd8 100644 --- a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/ConvertIREEToHAL.h +++ b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/ConvertUtilToHAL.h
@@ -4,8 +4,8 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef IREE_COMPILER_DIALECT_HAL_CONVERSION_CONVERTIREETOHAL_H_ -#define IREE_COMPILER_DIALECT_HAL_CONVERSION_CONVERTIREETOHAL_H_ +#ifndef IREE_COMPILER_DIALECT_HAL_CONVERSION_CONVERTUTILTOHAL_H_ +#define IREE_COMPILER_DIALECT_HAL_CONVERSION_CONVERTUTILTOHAL_H_ #include "mlir/IR/MLIRContext.h" #include "mlir/IR/PatternMatch.h" @@ -19,11 +19,11 @@ // Appends all patterns for lowering IREE ops to HAL buffer ops and sets their // legality. -void populateIREEToHALPatterns(MLIRContext *context, ConversionTarget &target, +void populateUtilToHALPatterns(MLIRContext *context, ConversionTarget &target, TypeConverter &typeConverter, OwningRewritePatternList &patterns); } // namespace iree_compiler } // namespace mlir -#endif // IREE_COMPILER_DIALECT_HAL_CONVERSION_CONVERTIREETOHAL_H_ +#endif // IREE_COMPILER_DIALECT_HAL_CONVERSION_CONVERTUTILTOHAL_H_
diff --git a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/test/BUILD b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/test/BUILD similarity index 100% rename from iree/compiler/Dialect/HAL/Conversion/IREEToHAL/test/BUILD rename to iree/compiler/Dialect/HAL/Conversion/UtilToHAL/test/BUILD
diff --git a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/test/CMakeLists.txt b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/test/CMakeLists.txt similarity index 93% rename from iree/compiler/Dialect/HAL/Conversion/IREEToHAL/test/CMakeLists.txt rename to iree/compiler/Dialect/HAL/Conversion/UtilToHAL/test/CMakeLists.txt index a602515..b372a39 100644 --- a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/test/CMakeLists.txt +++ b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/test/CMakeLists.txt
@@ -1,6 +1,6 @@ ################################################################################ # Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from # -# iree/compiler/Dialect/HAL/Conversion/IREEToHAL/test/BUILD # +# iree/compiler/Dialect/HAL/Conversion/UtilToHAL/test/BUILD # # # # Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary # # CMake-only content. #
diff --git a/iree/compiler/Dialect/HAL/Conversion/IREEToHAL/test/shape_constants.mlir b/iree/compiler/Dialect/HAL/Conversion/UtilToHAL/test/shape_constants.mlir similarity index 100% rename from iree/compiler/Dialect/HAL/Conversion/IREEToHAL/test/shape_constants.mlir rename to iree/compiler/Dialect/HAL/Conversion/UtilToHAL/test/shape_constants.mlir
diff --git a/iree/compiler/Dialect/HAL/Target/CUDA/BUILD b/iree/compiler/Dialect/HAL/Target/CUDA/BUILD index 282aea8..41236c5 100644 --- a/iree/compiler/Dialect/HAL/Target/CUDA/BUILD +++ b/iree/compiler/Dialect/HAL/Target/CUDA/BUILD
@@ -33,9 +33,11 @@ name = "CUDA", srcs = [ "CUDATarget.cpp", + "NoLoopUnrollPass.cpp", ], hdrs = [ "CUDATarget.h", + "LLVMPasses.h", ], deps = [ ":cuda_libdevice", @@ -45,6 +47,7 @@ "//iree/compiler/Dialect/HAL/Target", "//iree/compiler/Utils", "//iree/schemas:cuda_executable_def_c_fbs", + "@llvm-project//llvm:Analysis", "@llvm-project//llvm:BitReader", "@llvm-project//llvm:Core", "@llvm-project//llvm:IPO",
diff --git a/iree/compiler/Dialect/HAL/Target/CUDA/CMakeLists.txt b/iree/compiler/Dialect/HAL/Target/CUDA/CMakeLists.txt index 7ebad2b..c290659 100644 --- a/iree/compiler/Dialect/HAL/Target/CUDA/CMakeLists.txt +++ b/iree/compiler/Dialect/HAL/Target/CUDA/CMakeLists.txt
@@ -33,10 +33,13 @@ CUDA HDRS "CUDATarget.h" + "LLVMPasses.h" SRCS "CUDATarget.cpp" + "NoLoopUnrollPass.cpp" DEPS ::cuda_libdevice + LLVMAnalysis LLVMBitReader LLVMCore LLVMipo
diff --git a/iree/compiler/Dialect/HAL/Target/CUDA/CUDATarget.cpp b/iree/compiler/Dialect/HAL/Target/CUDA/CUDATarget.cpp index 2c0a276..1ce428e 100644 --- a/iree/compiler/Dialect/HAL/Target/CUDA/CUDATarget.cpp +++ b/iree/compiler/Dialect/HAL/Target/CUDA/CUDATarget.cpp
@@ -7,6 +7,7 @@ #include "iree/compiler/Dialect/HAL/Target/CUDA/CUDATarget.h" #include "iree/compiler/Codegen/Passes.h" +#include "iree/compiler/Dialect/HAL/Target/CUDA/LLVMPasses.h" #include "iree/compiler/Dialect/HAL/Target/CUDA/libdevice.h" #include "iree/compiler/Dialect/HAL/Target/TargetRegistry.h" #include "iree/compiler/Utils/FlatbufferUtils.h" @@ -44,6 +45,12 @@ llvm::raw_string_ostream stream(targetISA); llvm::buffer_ostream pstream(stream); llvm::legacy::PassManager codegenPasses; + // Workaround for CUDA driver bug + // (https://bugs.llvm.org/show_bug.cgi?id=48771), we mark all the loops with + // the no unroll metadata. This bug is fixed in cuda 11.4 but since we still + // run on older driver we need to keep it. + // TODO(thomasraoux): Remove it once we stop supporting older drivers. + codegenPasses.add(llvm::createSetNoUnrollPass()); targetMachine.addPassesToEmitFile(codegenPasses, pstream, nullptr, llvm::CGFT_AssemblyFile); codegenPasses.run(module);
diff --git a/iree/compiler/Dialect/HAL/Target/CUDA/LLVMPasses.h b/iree/compiler/Dialect/HAL/Target/CUDA/LLVMPasses.h new file mode 100644 index 0000000..cf997b1 --- /dev/null +++ b/iree/compiler/Dialect/HAL/Target/CUDA/LLVMPasses.h
@@ -0,0 +1,19 @@ +// Copyright 2021 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_COMPILER_DIALECT_HAL_TARGET_CUDA_PASS_H_ +#define IREE_COMPILER_DIALECT_HAL_TARGET_CUDA_PASS_H_ + +namespace llvm { + +class Pass; + +/// Pass to mark all loops with llvm metadata to disable unrolling. +Pass *createSetNoUnrollPass(); + +} // namespace llvm + +#endif // IREE_COMPILER_DIALECT_HAL_TARGET_CUDA_PASS_H_
diff --git a/iree/compiler/Dialect/HAL/Target/CUDA/NoLoopUnrollPass.cpp b/iree/compiler/Dialect/HAL/Target/CUDA/NoLoopUnrollPass.cpp new file mode 100644 index 0000000..dfb6059 --- /dev/null +++ b/iree/compiler/Dialect/HAL/Target/CUDA/NoLoopUnrollPass.cpp
@@ -0,0 +1,45 @@ +// Copyright 2021 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +#include "iree/compiler/Dialect/HAL/Target/CUDA/LLVMPasses.h" +#include "llvm/Analysis/LoopPass.h" +#include "llvm/InitializePasses.h" +#include "llvm/Pass.h" + +using namespace llvm; + +#define DEBUG_TYPE "iree-dialect-hal-cuda-llvm-nounroll" + +namespace llvm { +void initializeNoLoopUnrollPass(PassRegistry &Registry); +} + +namespace { +/// Pass that mark all loops with llvm.loop.unroll.disable metadata. +class NoLoopUnroll : public LoopPass { + public: + static char ID; + NoLoopUnroll() : LoopPass(ID) { + initializeNoLoopUnrollPass(*PassRegistry::getPassRegistry()); + } + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesCFG(); + } + bool runOnLoop(Loop *L, LPPassManager &LPM) override { + L->setLoopAlreadyUnrolled(); + return true; + } + StringRef getPassName() const override { return "Set Nounroll pass"; } +}; + +} // namespace + +char NoLoopUnroll::ID = 0; + +INITIALIZE_PASS_BEGIN(NoLoopUnroll, DEBUG_TYPE, "Set Nounroll", false, false) +INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass) +INITIALIZE_PASS_END(NoLoopUnroll, DEBUG_TYPE, "Set Nounroll", false, false) + +Pass *llvm::createSetNoUnrollPass() { return new NoLoopUnroll(); }
diff --git a/iree/compiler/Dialect/HAL/Transforms/BUILD b/iree/compiler/Dialect/HAL/Transforms/BUILD index ab73955..1d65681 100644 --- a/iree/compiler/Dialect/HAL/Transforms/BUILD +++ b/iree/compiler/Dialect/HAL/Transforms/BUILD
@@ -40,8 +40,8 @@ "//iree/compiler/Dialect/HAL/Conversion", "//iree/compiler/Dialect/HAL/Conversion/FlowToHAL", "//iree/compiler/Dialect/HAL/Conversion/HALToHAL", - "//iree/compiler/Dialect/HAL/Conversion/IREEToHAL", "//iree/compiler/Dialect/HAL/Conversion/StandardToHAL", + "//iree/compiler/Dialect/HAL/Conversion/UtilToHAL", "//iree/compiler/Dialect/HAL/IR", "//iree/compiler/Dialect/HAL/IR:HALDialect", "//iree/compiler/Dialect/HAL/Target",
diff --git a/iree/compiler/Dialect/HAL/Transforms/CMakeLists.txt b/iree/compiler/Dialect/HAL/Transforms/CMakeLists.txt index 101666c..795fcb6 100644 --- a/iree/compiler/Dialect/HAL/Transforms/CMakeLists.txt +++ b/iree/compiler/Dialect/HAL/Transforms/CMakeLists.txt
@@ -46,8 +46,8 @@ iree::compiler::Dialect::HAL::Conversion iree::compiler::Dialect::HAL::Conversion::FlowToHAL iree::compiler::Dialect::HAL::Conversion::HALToHAL - iree::compiler::Dialect::HAL::Conversion::IREEToHAL iree::compiler::Dialect::HAL::Conversion::StandardToHAL + iree::compiler::Dialect::HAL::Conversion::UtilToHAL iree::compiler::Dialect::HAL::IR iree::compiler::Dialect::HAL::IR::HALDialect iree::compiler::Dialect::HAL::Target
diff --git a/iree/compiler/Dialect/HAL/Transforms/ConvertToHAL.cpp b/iree/compiler/Dialect/HAL/Transforms/ConvertToHAL.cpp index a06b073..75ea007 100644 --- a/iree/compiler/Dialect/HAL/Transforms/ConvertToHAL.cpp +++ b/iree/compiler/Dialect/HAL/Transforms/ConvertToHAL.cpp
@@ -9,9 +9,9 @@ #include "iree/compiler/Dialect/HAL/Conversion/ConversionTarget.h" #include "iree/compiler/Dialect/HAL/Conversion/FlowToHAL/ConvertFlowToHAL.h" #include "iree/compiler/Dialect/HAL/Conversion/HALToHAL/ConvertHALToHAL.h" -#include "iree/compiler/Dialect/HAL/Conversion/IREEToHAL/ConvertIREEToHAL.h" #include "iree/compiler/Dialect/HAL/Conversion/StandardToHAL/ConvertStandardToHAL.h" #include "iree/compiler/Dialect/HAL/Conversion/TypeConverter.h" +#include "iree/compiler/Dialect/HAL/Conversion/UtilToHAL/ConvertUtilToHAL.h" #include "iree/compiler/Dialect/HAL/IR/HALDialect.h" #include "iree/compiler/Dialect/HAL/IR/HALOps.h" #include "iree/compiler/Dialect/HAL/IR/HALTypes.h" @@ -71,7 +71,7 @@ OwningRewritePatternList patterns(&getContext()); - populateIREEToHALPatterns(context, conversionTarget, typeConverter, + populateUtilToHALPatterns(context, conversionTarget, typeConverter, patterns); populateUtilConversionPatterns(context, conversionTarget, typeConverter,
diff --git a/iree/compiler/Dialect/Util/Conversion/ConversionPatterns.cpp b/iree/compiler/Dialect/Util/Conversion/ConversionPatterns.cpp index f800f8c..deeb85e 100644 --- a/iree/compiler/Dialect/Util/Conversion/ConversionPatterns.cpp +++ b/iree/compiler/Dialect/Util/Conversion/ConversionPatterns.cpp
@@ -20,18 +20,34 @@ namespace { //===----------------------------------------------------------------------===// -// Hints +// Utilities //===----------------------------------------------------------------------===// -class PreserveDoNotOptimize - : public OpConversionPattern<IREE::Util::DoNotOptimizeOp> { +template <typename T> +class GenericConvertTypesConversion : public OpConversionPattern<T> { public: - using OpConversionPattern<IREE::Util::DoNotOptimizeOp>::OpConversionPattern; + using OpConversionPattern<T>::OpConversionPattern; + LogicalResult matchAndRewrite( - IREE::Util::DoNotOptimizeOp op, llvm::ArrayRef<Value> operands, + T op, llvm::ArrayRef<Value> newOperands, ConversionPatternRewriter &rewriter) const override { - rewriter.replaceOpWithNewOp<IREE::Util::DoNotOptimizeOp>(op, operands, - op->getAttrs()); + SmallVector<Type> newTypes; + bool anyChanged = false; + for (auto oldNew : llvm::zip(op->getOperands(), newOperands)) { + auto oldValue = std::get<0>(oldNew); + auto newValue = std::get<1>(oldNew); + if (oldValue.getType() != newValue.getType()) { + anyChanged = true; + break; + } + } + for (auto oldType : op.getOperation()->getResultTypes()) { + auto newType = this->getTypeConverter()->convertType(oldType); + if (oldType != newType) anyChanged = true; + newTypes.push_back(newType); + } + if (!anyChanged) return failure(); + rewriter.replaceOpWithNewOp<T>(op, newTypes, newOperands, op->getAttrs()); return success(); } }; @@ -41,7 +57,17 @@ void populateUtilConversionPatterns(MLIRContext *context, TypeConverter &typeConverter, OwningRewritePatternList &patterns) { - patterns.insert<PreserveDoNotOptimize>(context); + patterns.insert<GenericConvertTypesConversion<IREE::Util::DoNotOptimizeOp>>( + typeConverter, context); + + typeConverter.addConversion([&](IREE::Util::ListType type) { + auto elementType = typeConverter.convertType(type.getElementType()); + return IREE::Util::ListType::get(elementType); + }); + patterns.insert<GenericConvertTypesConversion<IREE::Util::ListCreateOp>, + GenericConvertTypesConversion<IREE::Util::ListGetOp>, + GenericConvertTypesConversion<IREE::Util::ListSetOp>>( + typeConverter, context); } void populateUtilConversionPatterns(MLIRContext *context, @@ -55,6 +81,19 @@ }); }); + conversionTarget.addDynamicallyLegalOp<IREE::Util::ListCreateOp>( + [&](IREE::Util::ListCreateOp op) { + return typeConverter.isLegal(op.getType()); + }); + conversionTarget.addDynamicallyLegalOp<IREE::Util::ListGetOp>( + [&](IREE::Util::ListGetOp op) { + return typeConverter.isLegal(op.getType()); + }); + conversionTarget.addDynamicallyLegalOp<IREE::Util::ListSetOp>( + [&](IREE::Util::ListSetOp op) { + return typeConverter.isLegal(op.value().getType()); + }); + populateUtilConversionPatterns(context, typeConverter, patterns); }
diff --git a/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertListOps.cpp b/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertListOps.cpp index 3c846ce..97c26b5 100644 --- a/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertListOps.cpp +++ b/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertListOps.cpp
@@ -122,6 +122,7 @@ } // namespace void populateUtilListToVMPatterns(MLIRContext *context, + ConversionTarget &conversionTarget, TypeConverter &typeConverter, OwningRewritePatternList &patterns) { typeConverter.addConversion( @@ -135,6 +136,11 @@ if (!elementType) return llvm::None; return IREE::VM::RefType::get(IREE::VM::ListType::get(elementType)); }); + + conversionTarget.addIllegalOp< + IREE::Util::ListCreateOp, IREE::Util::ListSizeOp, + IREE::Util::ListResizeOp, IREE::Util::ListGetOp, IREE::Util::ListSetOp>(); + patterns .insert<ListCreateOpConversion, ListSizeOpConversion, ListResizeOpConversion, ListGetOpConversion, ListSetOpConversion>(
diff --git a/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertUtilToVM.cpp b/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertUtilToVM.cpp index f4d09ee..dd9a5c9 100644 --- a/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertUtilToVM.cpp +++ b/iree/compiler/Dialect/VM/Conversion/UtilToVM/ConvertUtilToVM.cpp
@@ -25,6 +25,7 @@ TypeConverter &typeConverter, OwningRewritePatternList &patterns); void populateUtilListToVMPatterns(MLIRContext *context, + ConversionTarget &conversionTarget, TypeConverter &typeConverter, OwningRewritePatternList &patterns); void populateUtilStatusToVMPatterns(MLIRContext *context, @@ -114,7 +115,8 @@ populateUtilGlobalToVMPatterns(context, conversionTarget, typeConverter, patterns); - populateUtilListToVMPatterns(context, typeConverter, patterns); + populateUtilListToVMPatterns(context, conversionTarget, typeConverter, + patterns); populateUtilStatusToVMPatterns(context, conversionTarget, typeConverter, patterns); }
diff --git a/iree/compiler/Dialect/VM/Transforms/Conversion.cpp b/iree/compiler/Dialect/VM/Transforms/Conversion.cpp index 77e686c..fe293b3 100644 --- a/iree/compiler/Dialect/VM/Transforms/Conversion.cpp +++ b/iree/compiler/Dialect/VM/Transforms/Conversion.cpp
@@ -10,6 +10,7 @@ #include "iree/compiler/Dialect/Shape/IR/ShapeOps.h" #include "iree/compiler/Dialect/Util/Conversion/ConversionPatterns.h" #include "iree/compiler/Dialect/Util/IR/UtilDialect.h" +#include "iree/compiler/Dialect/Util/IR/UtilOps.h" #include "iree/compiler/Dialect/VM/Conversion/ConversionDialectInterface.h" #include "iree/compiler/Dialect/VM/Conversion/ConversionTarget.h" #include "iree/compiler/Dialect/VM/Conversion/ImportUtils.h" @@ -129,6 +130,8 @@ } OwningRewritePatternList conversionPatterns(&getContext()); + populateUtilConversionPatterns(context, conversionTarget, typeConverter, + conversionPatterns); populateUtilToVMPatterns(context, conversionTarget, typeConverter, conversionPatterns); populateStandardToVMPatterns(context, typeConverter, conversionPatterns); @@ -150,8 +153,6 @@ importSymbols, conversionPatterns, typeConverter); } Shape::populateFoldConversionPatterns(context, conversionPatterns); - populateUtilConversionPatterns(context, conversionTarget, typeConverter, - conversionPatterns); if (failed(applyPartialConversion(outerModuleOp, conversionTarget, std::move(conversionPatterns)))) {
diff --git a/iree/hal/dylib/registration/BUILD b/iree/hal/dylib/registration/BUILD index 3c2e252..3c0dce7 100644 --- a/iree/hal/dylib/registration/BUILD +++ b/iree/hal/dylib/registration/BUILD
@@ -40,7 +40,7 @@ "//iree/hal/local", "//iree/hal/local:task_driver", "//iree/hal/local/loaders:embedded_library_loader", - "//iree/hal/local/loaders:legacy_library_loader", + "//iree/hal/local/loaders:system_library_loader", "//iree/task:api", ], ) @@ -57,6 +57,6 @@ "//iree/hal", "//iree/hal/local", "//iree/hal/local:sync_driver", - "//iree/hal/local/loaders:legacy_library_loader", + "//iree/hal/local/loaders:system_library_loader", ], )
diff --git a/iree/hal/dylib/registration/CMakeLists.txt b/iree/hal/dylib/registration/CMakeLists.txt index 83aface..03e3ea5 100644 --- a/iree/hal/dylib/registration/CMakeLists.txt +++ b/iree/hal/dylib/registration/CMakeLists.txt
@@ -32,7 +32,7 @@ iree::hal iree::hal::local iree::hal::local::loaders::embedded_library_loader - iree::hal::local::loaders::legacy_library_loader + iree::hal::local::loaders::system_library_loader iree::hal::local::task_driver iree::task::api DEFINES @@ -51,7 +51,7 @@ iree::base iree::hal iree::hal::local - iree::hal::local::loaders::legacy_library_loader + iree::hal::local::loaders::system_library_loader iree::hal::local::sync_driver DEFINES "IREE_HAL_HAVE_DYLIB_SYNC_DRIVER_MODULE=1"
diff --git a/iree/hal/dylib/registration/driver_module.c b/iree/hal/dylib/registration/driver_module.c index 3f17988..16fe55f 100644 --- a/iree/hal/dylib/registration/driver_module.c +++ b/iree/hal/dylib/registration/driver_module.c
@@ -13,7 +13,7 @@ #include "iree/base/internal/flags.h" #include "iree/hal/local/executable_loader.h" #include "iree/hal/local/loaders/embedded_library_loader.h" -#include "iree/hal/local/loaders/legacy_library_loader.h" +#include "iree/hal/local/loaders/system_library_loader.h" #include "iree/hal/local/task_device.h" #include "iree/hal/local/task_driver.h" #include "iree/task/api.h" @@ -66,7 +66,7 @@ &loaders[loader_count++]); } if (iree_status_is_ok(status)) { - status = iree_hal_legacy_library_loader_create( + status = iree_hal_system_library_loader_create( iree_hal_executable_import_provider_null(), allocator, &loaders[loader_count++]); }
diff --git a/iree/hal/dylib/registration/driver_module_sync.c b/iree/hal/dylib/registration/driver_module_sync.c index 0ac89c3..414a708 100644 --- a/iree/hal/dylib/registration/driver_module_sync.c +++ b/iree/hal/dylib/registration/driver_module_sync.c
@@ -11,7 +11,7 @@ #include "iree/base/api.h" #include "iree/hal/local/executable_loader.h" -#include "iree/hal/local/loaders/legacy_library_loader.h" +#include "iree/hal/local/loaders/system_library_loader.h" #include "iree/hal/local/sync_device.h" #include "iree/hal/local/sync_driver.h" @@ -49,7 +49,7 @@ iree_hal_sync_device_params_initialize(&default_params); iree_hal_executable_loader_t* dylib_loader = NULL; - iree_status_t status = iree_hal_legacy_library_loader_create( + iree_status_t status = iree_hal_system_library_loader_create( iree_hal_executable_import_provider_null(), allocator, &dylib_loader); iree_hal_executable_loader_t* loaders[1] = {dylib_loader};
diff --git a/iree/hal/local/loaders/BUILD b/iree/hal/local/loaders/BUILD index 28b9cb4..3d54ae9 100644 --- a/iree/hal/local/loaders/BUILD +++ b/iree/hal/local/loaders/BUILD
@@ -34,26 +34,6 @@ ) cc_library( - name = "legacy_library_loader", - srcs = ["legacy_library_loader.c"], - hdrs = ["legacy_library_loader.h"], - defines = [ - "IREE_HAL_HAVE_LEGACY_LIBRARY_LOADER=1", - ], - deps = [ - "//iree/base", - "//iree/base:core_headers", - "//iree/base:tracing", - "//iree/base/internal:dynamic_library", - "//iree/base/internal:flatcc", - "//iree/hal", - "//iree/hal/local", - "//iree/hal/local:executable_library", - "//iree/schemas:dylib_executable_def_c_fbs", - ], -) - -cc_library( name = "static_library_loader", srcs = ["static_library_loader.c"], hdrs = ["static_library_loader.h"], @@ -69,6 +49,26 @@ ], ) +cc_library( + name = "system_library_loader", + srcs = ["system_library_loader.c"], + hdrs = ["system_library_loader.h"], + defines = [ + "IREE_HAL_HAVE_SYSTEM_LIBRARY_LOADER=1", + ], + deps = [ + "//iree/base", + "//iree/base:core_headers", + "//iree/base:tracing", + "//iree/base/internal:dynamic_library", + "//iree/base/internal:flatcc", + "//iree/hal", + "//iree/hal/local", + "//iree/hal/local:executable_library", + "//iree/schemas:dylib_executable_def_c_fbs", + ], +) + iree_cmake_extra_content( content = """ if(${IREE_HAL_DRIVER_VMVX})
diff --git a/iree/hal/local/loaders/CMakeLists.txt b/iree/hal/local/loaders/CMakeLists.txt index c55fcee..fe5dd54 100644 --- a/iree/hal/local/loaders/CMakeLists.txt +++ b/iree/hal/local/loaders/CMakeLists.txt
@@ -32,28 +32,6 @@ iree_cc_library( NAME - legacy_library_loader - HDRS - "legacy_library_loader.h" - SRCS - "legacy_library_loader.c" - DEPS - iree::base - iree::base::core_headers - iree::base::internal::dynamic_library - iree::base::internal::flatcc - iree::base::tracing - iree::hal - iree::hal::local - iree::hal::local::executable_library - iree::schemas::dylib_executable_def_c_fbs - DEFINES - "IREE_HAL_HAVE_LEGACY_LIBRARY_LOADER=1" - PUBLIC -) - -iree_cc_library( - NAME static_library_loader HDRS "static_library_loader.h" @@ -70,6 +48,28 @@ PUBLIC ) +iree_cc_library( + NAME + system_library_loader + HDRS + "system_library_loader.h" + SRCS + "system_library_loader.c" + DEPS + iree::base + iree::base::core_headers + iree::base::internal::dynamic_library + iree::base::internal::flatcc + iree::base::tracing + iree::hal + iree::hal::local + iree::hal::local::executable_library + iree::schemas::dylib_executable_def_c_fbs + DEFINES + "IREE_HAL_HAVE_SYSTEM_LIBRARY_LOADER=1" + PUBLIC +) + if(${IREE_HAL_DRIVER_VMVX}) iree_cc_library(
diff --git a/iree/hal/local/loaders/legacy_library_loader.c b/iree/hal/local/loaders/system_library_loader.c similarity index 85% rename from iree/hal/local/loaders/legacy_library_loader.c rename to iree/hal/local/loaders/system_library_loader.c index fa9fa1f..49f7337 100644 --- a/iree/hal/local/loaders/legacy_library_loader.c +++ b/iree/hal/local/loaders/system_library_loader.c
@@ -4,7 +4,7 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#include "iree/hal/local/loaders/legacy_library_loader.h" +#include "iree/hal/local/loaders/system_library_loader.h" #include <stdbool.h> #include <stddef.h> @@ -61,10 +61,10 @@ } //===----------------------------------------------------------------------===// -// iree_hal_legacy_executable_t +// iree_hal_system_executable_t //===----------------------------------------------------------------------===// -typedef struct iree_hal_legacy_executable_t { +typedef struct iree_hal_system_executable_t { iree_hal_local_executable_t base; // Flatbuffer definition referencing the executable memory. @@ -83,13 +83,13 @@ } library; iree_hal_local_executable_layout_t* layouts[]; -} iree_hal_legacy_executable_t; +} iree_hal_system_executable_t; extern const iree_hal_local_executable_vtable_t - iree_hal_legacy_executable_vtable; + iree_hal_system_executable_vtable; -static iree_status_t iree_hal_legacy_executable_extract_and_load( - iree_hal_legacy_executable_t* executable, iree_allocator_t host_allocator) { +static iree_status_t iree_hal_system_executable_extract_and_load( + iree_hal_system_executable_t* executable, iree_allocator_t host_allocator) { flatbuffers_uint8_vec_t embedded_library_vec = iree_DyLibExecutableDef_library_embedded_get(executable->def); IREE_RETURN_IF_ERROR(iree_dynamic_library_load_from_memory( @@ -114,8 +114,8 @@ return iree_ok_status(); } -static iree_status_t iree_hal_legacy_executable_query_library( - iree_hal_legacy_executable_t* executable) { +static iree_status_t iree_hal_system_executable_query_library( + iree_hal_system_executable_t* executable) { // Get the exported symbol used to get the library metadata. iree_hal_executable_library_query_fn_t query_fn = NULL; IREE_RETURN_IF_ERROR(iree_dynamic_library_lookup_symbol( @@ -170,15 +170,15 @@ return iree_ok_status(); } -static int iree_hal_legacy_executable_import_thunk_v0( +static int iree_hal_system_executable_import_thunk_v0( iree_hal_executable_import_v0_t fn_ptr, void* import_params) { return fn_ptr(import_params); } // Resolves all of the imports declared by the executable using the given // |import_provider|. -static iree_status_t iree_hal_legacy_executable_resolve_imports( - iree_hal_legacy_executable_t* executable, +static iree_status_t iree_hal_system_executable_resolve_imports( + iree_hal_system_executable_t* executable, const iree_hal_executable_import_provider_t import_provider) { const iree_hal_executable_import_table_v0_t* import_table = &executable->library.v0->imports; @@ -186,7 +186,7 @@ IREE_TRACE_ZONE_BEGIN(z0); // Pass all imports right through. - executable->base.import_thunk = iree_hal_legacy_executable_import_thunk_v0; + executable->base.import_thunk = iree_hal_system_executable_import_thunk_v0; // Allocate storage for the imports. IREE_RETURN_AND_END_ZONE_IF_ERROR( @@ -211,7 +211,7 @@ return iree_ok_status(); } -static iree_status_t iree_hal_legacy_executable_create( +static iree_status_t iree_hal_system_executable_create( iree_DyLibExecutableDef_table_t executable_def, iree_host_size_t executable_layout_count, iree_hal_executable_layout_t* const* executable_layouts, @@ -223,7 +223,7 @@ *out_executable = NULL; IREE_TRACE_ZONE_BEGIN(z0); - iree_hal_legacy_executable_t* executable = NULL; + iree_hal_system_executable_t* executable = NULL; iree_host_size_t total_size = sizeof(*executable) + executable_layout_count * sizeof(*executable->layouts); @@ -231,7 +231,7 @@ iree_allocator_malloc(host_allocator, total_size, (void**)&executable); if (iree_status_is_ok(status)) { iree_hal_local_executable_initialize( - &iree_hal_legacy_executable_vtable, executable_layout_count, + &iree_hal_system_executable_vtable, executable_layout_count, executable_layouts, &executable->layouts[0], host_allocator, &executable->base); executable->def = executable_def; @@ -242,16 +242,16 @@ // This is bad, but ehh all this is getting deleted soon and hopefully we // can avoid ever touching the disk at all. status = - iree_hal_legacy_executable_extract_and_load(executable, host_allocator); + iree_hal_system_executable_extract_and_load(executable, host_allocator); } if (iree_status_is_ok(status)) { // Query metadata and get the entry point function pointers. - status = iree_hal_legacy_executable_query_library(executable); + status = iree_hal_system_executable_query_library(executable); } if (iree_status_is_ok(status)) { // Resolve imports, if any. status = - iree_hal_legacy_executable_resolve_imports(executable, import_provider); + iree_hal_system_executable_resolve_imports(executable, import_provider); } if (iree_status_is_ok(status)) { // Check to make sure that the entry point count matches the layouts @@ -274,10 +274,10 @@ return status; } -static void iree_hal_legacy_executable_destroy( +static void iree_hal_system_executable_destroy( iree_hal_executable_t* base_executable) { - iree_hal_legacy_executable_t* executable = - (iree_hal_legacy_executable_t*)base_executable; + iree_hal_system_executable_t* executable = + (iree_hal_system_executable_t*)base_executable; iree_allocator_t host_allocator = executable->base.host_allocator; IREE_TRACE_ZONE_BEGIN(z0); @@ -290,12 +290,12 @@ IREE_TRACE_ZONE_END(z0); } -static iree_status_t iree_hal_legacy_executable_issue_call( +static iree_status_t iree_hal_system_executable_issue_call( iree_hal_local_executable_t* base_executable, iree_host_size_t ordinal, const iree_hal_executable_dispatch_state_v0_t* dispatch_state, const iree_hal_vec3_t* workgroup_id, iree_byte_span_t local_memory) { - iree_hal_legacy_executable_t* executable = - (iree_hal_legacy_executable_t*)base_executable; + iree_hal_system_executable_t* executable = + (iree_hal_system_executable_t*)base_executable; const iree_hal_executable_library_v0_t* library = executable->library.v0; if (IREE_UNLIKELY(ordinal >= library->exports.count)) { @@ -334,27 +334,27 @@ ret); } -const iree_hal_local_executable_vtable_t iree_hal_legacy_executable_vtable = { - /*.base=*/ - { - /*.destroy=*/iree_hal_legacy_executable_destroy, - }, - /*.issue_call=*/iree_hal_legacy_executable_issue_call, +const iree_hal_local_executable_vtable_t iree_hal_system_executable_vtable = { + .base = + { + .destroy = iree_hal_system_executable_destroy, + }, + .issue_call = iree_hal_system_executable_issue_call, }; //===----------------------------------------------------------------------===// -// iree_hal_legacy_library_loader_t +// iree_hal_system_library_loader_t //===----------------------------------------------------------------------===// -typedef struct iree_hal_legacy_library_loader_t { +typedef struct iree_hal_system_library_loader_t { iree_hal_executable_loader_t base; iree_allocator_t host_allocator; -} iree_hal_legacy_library_loader_t; +} iree_hal_system_library_loader_t; extern const iree_hal_executable_loader_vtable_t - iree_hal_legacy_library_loader_vtable; + iree_hal_system_library_loader_vtable; -iree_status_t iree_hal_legacy_library_loader_create( +iree_status_t iree_hal_system_library_loader_create( iree_hal_executable_import_provider_t import_provider, iree_allocator_t host_allocator, iree_hal_executable_loader_t** out_executable_loader) { @@ -362,12 +362,12 @@ *out_executable_loader = NULL; IREE_TRACE_ZONE_BEGIN(z0); - iree_hal_legacy_library_loader_t* executable_loader = NULL; + iree_hal_system_library_loader_t* executable_loader = NULL; iree_status_t status = iree_allocator_malloc( host_allocator, sizeof(*executable_loader), (void**)&executable_loader); if (iree_status_is_ok(status)) { iree_hal_executable_loader_initialize( - &iree_hal_legacy_library_loader_vtable, import_provider, + &iree_hal_system_library_loader_vtable, import_provider, &executable_loader->base); executable_loader->host_allocator = host_allocator; *out_executable_loader = (iree_hal_executable_loader_t*)executable_loader; @@ -377,10 +377,10 @@ return status; } -static void iree_hal_legacy_library_loader_destroy( +static void iree_hal_system_library_loader_destroy( iree_hal_executable_loader_t* base_executable_loader) { - iree_hal_legacy_library_loader_t* executable_loader = - (iree_hal_legacy_library_loader_t*)base_executable_loader; + iree_hal_system_library_loader_t* executable_loader = + (iree_hal_system_library_loader_t*)base_executable_loader; iree_allocator_t host_allocator = executable_loader->host_allocator; IREE_TRACE_ZONE_BEGIN(z0); @@ -397,7 +397,7 @@ #define IREE_PLATFORM_DYLIB_TYPE "elf" #endif // IREE_PLATFORM_* -static bool iree_hal_legacy_library_loader_query_support( +static bool iree_hal_system_library_loader_query_support( iree_hal_executable_loader_t* base_executable_loader, iree_hal_executable_caching_mode_t caching_mode, iree_string_view_t executable_format) { @@ -406,12 +406,12 @@ iree_make_cstring_view("system-" IREE_PLATFORM_DYLIB_TYPE "-" IREE_ARCH)); } -static iree_status_t iree_hal_legacy_library_loader_try_load( +static iree_status_t iree_hal_system_library_loader_try_load( iree_hal_executable_loader_t* base_executable_loader, const iree_hal_executable_spec_t* executable_spec, iree_hal_executable_t** out_executable) { - iree_hal_legacy_library_loader_t* executable_loader = - (iree_hal_legacy_library_loader_t*)base_executable_loader; + iree_hal_system_library_loader_t* executable_loader = + (iree_hal_system_library_loader_t*)base_executable_loader; IREE_TRACE_ZONE_BEGIN(z0); // Verify and fetch the executable flatbuffer wrapper. @@ -423,7 +423,7 @@ // Perform the load (and requisite disgusting hackery). IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_hal_legacy_executable_create( + z0, iree_hal_system_executable_create( executable_def, executable_spec->executable_layout_count, executable_spec->executable_layouts, base_executable_loader->import_provider, @@ -434,8 +434,8 @@ } const iree_hal_executable_loader_vtable_t - iree_hal_legacy_library_loader_vtable = { - /*.destroy=*/iree_hal_legacy_library_loader_destroy, - /*.query_support=*/iree_hal_legacy_library_loader_query_support, - /*.try_load=*/iree_hal_legacy_library_loader_try_load, + iree_hal_system_library_loader_vtable = { + .destroy = iree_hal_system_library_loader_destroy, + .query_support = iree_hal_system_library_loader_query_support, + .try_load = iree_hal_system_library_loader_try_load, };
diff --git a/iree/hal/local/loaders/legacy_library_loader.h b/iree/hal/local/loaders/system_library_loader.h similarity index 81% rename from iree/hal/local/loaders/legacy_library_loader.h rename to iree/hal/local/loaders/system_library_loader.h index 4374d87..23ffdd0 100644 --- a/iree/hal/local/loaders/legacy_library_loader.h +++ b/iree/hal/local/loaders/system_library_loader.h
@@ -4,8 +4,8 @@ // See https://llvm.org/LICENSE.txt for license information. // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -#ifndef IREE_HAL_LOCAL_LOADERS_LEGACY_LIBRARY_LOADER_H_ -#define IREE_HAL_LOCAL_LOADERS_LEGACY_LIBRARY_LOADER_H_ +#ifndef IREE_HAL_LOCAL_LOADERS_SYSTEM_LIBRARY_LOADER_H_ +#define IREE_HAL_LOCAL_LOADERS_SYSTEM_LIBRARY_LOADER_H_ #include <stdbool.h> #include <stdint.h> @@ -23,7 +23,7 @@ // This uses the legacy "dylib"-style format that will be deleted soon and is // only a placeholder until the compiler can be switched to output // iree_hal_executable_library_t-compatible files. -iree_status_t iree_hal_legacy_library_loader_create( +iree_status_t iree_hal_system_library_loader_create( iree_hal_executable_import_provider_t import_provider, iree_allocator_t host_allocator, iree_hal_executable_loader_t** out_executable_loader); @@ -32,4 +32,4 @@ } // extern "C" #endif // __cplusplus -#endif // IREE_HAL_LOCAL_LOADERS_LEGACY_LIBRARY_LOADER_H_ +#endif // IREE_HAL_LOCAL_LOADERS_SYSTEM_LIBRARY_LOADER_H_
diff --git a/iree/test/e2e/models/BUILD b/iree/test/e2e/models/BUILD index 976a358..1732ac0 100644 --- a/iree/test/e2e/models/BUILD +++ b/iree/test/e2e/models/BUILD
@@ -72,3 +72,20 @@ driver = "vulkan", target_backend = "vulkan-spirv", ) + +iree_check_single_backend_test_suite( + name = "check_linalg_on_tensors_cuda_cuda", + timeout = "long", + srcs = CHECK_FRAMEWORK_TESTS, + compiler_flags = ["-iree-input-type=mhlo"], + driver = "cuda", + tags = [ + # CUDA cuInit fails with sanitizer on. + "noasan", + "nomsan", + "notsan", + "noubsan", + "requires-gpu-nvidia", + ], + target_backend = "cuda", +)
diff --git a/iree/test/e2e/models/CMakeLists.txt b/iree/test/e2e/models/CMakeLists.txt index e8d3983..3975477 100644 --- a/iree/test/e2e/models/CMakeLists.txt +++ b/iree/test/e2e/models/CMakeLists.txt
@@ -57,4 +57,24 @@ "-iree-input-type=mhlo" ) +iree_check_single_backend_test_suite( + NAME + check_linalg_on_tensors_cuda_cuda + SRCS + "bert_encoder_unrolled_fake_weights.mlir" + "mobilenetv3_fake_weights.mlir" + TARGET_BACKEND + "cuda" + DRIVER + "cuda" + COMPILER_FLAGS + "-iree-input-type=mhlo" + LABELS + "noasan" + "nomsan" + "notsan" + "noubsan" + "requires-gpu-nvidia" +) + ### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###