Finally moving VM type registration to iree_vm_instance_t. (#12650)
This allows for thread-safe type registration scoped to instances and
unregistration of types as required by types in dynamically loaded
modules that may wink out of existence at some point. The main trick
here was changing the type ID from an ordinal in the type table to just
the pointer of the type descriptor. This requires an extra 4 bytes per
ref on 64-bit systems but who cares - now there's no round-tripping
through the type table for common operations.
As part of simplifying the way types are referenced VM type descriptors
are now hidden behind iree_vm_ref_type_t. This makes refs much easier to
work with as there's only one way to reference types and it always
bottoms out on the registered descriptor
handle. It also allows us to remove some type descriptor indirection
we'd previously required in order to get reference counter offsets as we
can share the same packed type identifier in type defs, refs, or lists.
Thanks to @simon-camp for the required EmitC changes!
---------
Co-authored-by: Simon Camphausen <simon.camphausen@iml.fraunhofer.de>
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 8a7de15..ff54d02 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -361,11 +361,11 @@
if(IREE_BUILD_COMPILER)
message(STATUS "IREE compiler output formats:")
if(IREE_OUTPUT_FORMAT_C)
- message(STATUS " - C source module")
+ message(STATUS " - 'vm-c': textual C source module")
endif()
# The 'vm-bytecode' and 'vm-asm' formats are always enabled.
- message(STATUS " - VM Bytecode")
- message(STATUS " - VM MLIR Assembly")
+ message(STATUS " - 'vm-bytecode': VM bytecode")
+ message(STATUS " - 'vm-asm': VM MLIR assembly")
endif()
#-------------------------------------------------------------------------------
diff --git a/compiler/src/iree/compiler/ConstEval/Runtime.cpp b/compiler/src/iree/compiler/ConstEval/Runtime.cpp
index 7ba471a..8591b66 100644
--- a/compiler/src/iree/compiler/ConstEval/Runtime.cpp
+++ b/compiler/src/iree/compiler/ConstEval/Runtime.cpp
@@ -83,9 +83,9 @@
} // namespace
-CompiledBinary::CompiledBinary() {}
+CompiledBinary::CompiledBinary() = default;
-CompiledBinary::~CompiledBinary() {}
+CompiledBinary::~CompiledBinary() = default;
void CompiledBinary::deinitialize() {
hal_module.reset();
@@ -106,10 +106,10 @@
}
iree::vm::ref<iree_vm_list_t> inputs;
- IREE_CHECK_OK(iree_vm_list_create(/*element_type=*/nullptr, 0,
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 0,
iree_allocator_system(), &inputs));
iree::vm::ref<iree_vm_list_t> outputs;
- IREE_CHECK_OK(iree_vm_list_create(/*element_type=*/nullptr, 1,
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 1,
iree_allocator_system(), &outputs));
if (auto status =
@@ -188,7 +188,7 @@
auto context = loc.getContext();
Builder builder(context);
if (iree_vm_variant_is_value(variant)) {
- switch (variant.type.value_type) {
+ switch (iree_vm_type_def_as_value(variant.type)) {
case IREE_VM_VALUE_TYPE_I8:
return builder.getI8IntegerAttr(variant.i8);
case IREE_VM_VALUE_TYPE_I16:
@@ -203,7 +203,8 @@
return builder.getF64FloatAttr(variant.f64);
default:
emitError(loc) << "unrecognized evaluated value type: "
- << static_cast<int>(variant.type.value_type);
+ << static_cast<int>(
+ iree_vm_type_def_as_value(variant.type));
return {};
}
}
@@ -248,7 +249,7 @@
return convertedAttr;
} else {
iree_string_view_t typeName =
- iree_vm_ref_type_name(variant.type.ref_type);
+ iree_vm_ref_type_name(iree_vm_type_def_as_ref(variant.type));
emitError(loc) << "unrecognized evaluated ref type: "
<< StringRef(typeName.data, typeName.size);
return {};
@@ -311,7 +312,8 @@
IREE_CHECK_OK(
iree_hal_driver_registry_allocate(iree_allocator_system(), ®istry));
IREE_CHECK_OK(iree_hal_local_task_driver_module_register(registry));
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance));
IREE_CHECK_OK(iree_hal_module_register_all_types(instance.get()));
}
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/BUILD.bazel b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/BUILD.bazel
index 1c20632..6387084 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/BUILD.bazel
+++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/BUILD.bazel
@@ -41,6 +41,7 @@
"//compiler/src/iree/compiler/Dialect/VM/Analysis",
"//compiler/src/iree/compiler/Dialect/VM/IR",
"//compiler/src/iree/compiler/Dialect/VM/Utils:CallingConvention",
+ "//compiler/src/iree/compiler/Dialect/VM/Utils:TypeTable",
"@llvm-project//llvm:Support",
"@llvm-project//mlir:ArithDialect",
"@llvm-project//mlir:ControlFlowDialect",
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/CMakeLists.txt b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/CMakeLists.txt
index bf5491b..679bc89 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/CMakeLists.txt
@@ -43,6 +43,7 @@
iree::compiler::Dialect::VM::Analysis
iree::compiler::Dialect::VM::IR
iree::compiler::Dialect::VM::Utils::CallingConvention
+ iree::compiler::Dialect::VM::Utils::TypeTable
PUBLIC
)
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/ConvertVMToEmitC.cpp b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/ConvertVMToEmitC.cpp
index 56d0957..7191ab5 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/ConvertVMToEmitC.cpp
+++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/ConvertVMToEmitC.cpp
@@ -267,10 +267,11 @@
return result;
}
-std::optional<emitc::ApplyOp> createVmTypeDefPtr(
- ConversionPatternRewriter &rewriter, Operation *srcOp, Type elementType) {
- auto ctx = srcOp->getContext();
- auto loc = srcOp->getLoc();
+std::optional<Value> createVmTypeDefPtr(
+ ConversionPatternRewriter &rewriter, Location loc,
+ IREE::VM::EmitCTypeConverter &typeConverter, IREE::VM::ModuleOp moduleOp,
+ BlockArgument moduleArg, Type elementType) {
+ auto ctx = rewriter.getContext();
// Map from type to enum values of type iree_vm_value_type_t and
// iree_vm_ref_type_t
@@ -287,81 +288,64 @@
{"IREE_VM_VALUE_TYPE_F32", "IREE_VM_REF_TYPE_NULL"}},
{Float64Type::get(ctx),
{"IREE_VM_VALUE_TYPE_F64", "IREE_VM_REF_TYPE_NULL"}},
- {IREE::VM::OpaqueType::get(ctx),
- {"IREE_VM_VALUE_TYPE_NONE", "IREE_VM_REF_TYPE_NULL"}},
};
- Value elementTypeValue = emitc_builders::allocateVariable(
- rewriter, loc, emitc::OpaqueType::get(ctx, "iree_vm_type_def_t"));
-
- Value elementTypePtr =
- emitc_builders::addressOf(rewriter, loc, elementTypeValue);
-
- if (failed(clearStruct(rewriter, elementTypePtr))) {
- return std::nullopt;
- }
+ Value elementTypeValue;
auto ptr = valueTypeMap.find((elementType));
if (ptr != valueTypeMap.end()) {
- emitc_builders::structMemberAssign(rewriter, loc,
- /*memberName=*/"value_type",
- /*operand=*/elementTypeValue,
- /*value=*/ptr->second.first);
-
- emitc_builders::structMemberAssign(rewriter, loc,
- /*memberName=*/"ref_type",
- /*operand=*/elementTypeValue,
- /*value=*/ptr->second.second);
- } else {
- if (!elementType.isa<IREE::VM::RefType>()) {
- return std::nullopt;
- }
+ elementTypeValue =
+ rewriter
+ .create<emitc::CallOp>(
+ /*location=*/loc,
+ /*type=*/emitc::OpaqueType::get(ctx, "iree_vm_type_def_t"),
+ /*callee=*/StringAttr::get(ctx, "iree_vm_make_value_type_def"),
+ /*args=*/
+ ArrayAttr::get(
+ ctx, {emitc::OpaqueAttr::get(ctx, ptr->second.first)}),
+ /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{})
+ .getResult(0);
+ } else if (elementType.isa<IREE::VM::RefType>()) {
Type objType = elementType.cast<IREE::VM::RefType>().getObjectType();
- std::string typeName;
-
- if (objType.isa<IREE::VM::ListType>()) {
- typeName = "!vm.list";
- } else {
- llvm::raw_string_ostream sstream(typeName);
- objType.print(sstream);
- sstream.flush();
+ Type typeRefType = emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t");
+ Type typeRefArrayType = emitc::PointerType::get(typeRefType);
+ Optional<size_t> typeIndex = typeConverter.lookupType(objType);
+ if (!typeIndex.has_value()) {
+ moduleOp.emitError("type index lookup failed");
+ return std::nullopt;
}
- // Remove leading '!' and wrap in quotes
- typeName = std::string("\"") + typeName.substr(1) + std::string("\"");
+ Value typeArray = emitc_builders::structPtrMember(
+ rewriter, loc, typeRefArrayType, "types", moduleArg);
+ Value refType = emitc_builders::arrayElement(rewriter, loc, typeRefType,
+ typeIndex.value(), typeArray);
- auto typeNameCStringView = rewriter.create<emitc::CallOp>(
- /*location=*/loc,
- /*type=*/emitc::OpaqueType::get(ctx, "iree_string_view_t"),
- /*callee=*/StringAttr::get(ctx, "iree_make_cstring_view"),
- /*args=*/ArrayAttr::get(ctx, {emitc::OpaqueAttr::get(ctx, typeName)}),
- /*templateArgs=*/ArrayAttr{},
- /*operands=*/ArrayRef<Value>{});
-
- auto typeDescriptor = rewriter.create<emitc::CallOp>(
- /*location=*/loc,
- /*type=*/
- emitc::PointerType::get(
- emitc::OpaqueType::get(ctx, "const iree_vm_ref_type_descriptor_t")),
- /*callee=*/StringAttr::get(ctx, "iree_vm_ref_lookup_registered_type"),
- /*args=*/ArrayAttr{},
- /*templateArgs=*/ArrayAttr{},
- /*operands=*/ArrayRef<Value>{typeNameCStringView.getResult(0)});
-
- // TODDO(simon-camp) typeDescriptor might be NULL
- auto typeDescriptorType = emitc_builders::structPtrMember(
- rewriter, loc,
- /*type=*/emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t"),
- /*memberName=*/"type",
- /*operand=*/typeDescriptor.getResult(0));
-
- emitc_builders::structMemberAssign(rewriter, loc,
- /*memberName=*/"ref_type",
- /*operand=*/elementTypeValue,
- /*value=*/typeDescriptorType);
+ elementTypeValue =
+ rewriter
+ .create<emitc::CallOp>(
+ /*location=*/loc,
+ /*type=*/emitc::OpaqueType::get(ctx, "iree_vm_type_def_t"),
+ /*callee=*/StringAttr::get(ctx, "iree_vm_make_ref_type_def"),
+ /*args=*/ArrayAttr{},
+ /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{refType})
+ .getResult(0);
+ } else if (elementType.isa<IREE::VM::OpaqueType>()) {
+ elementTypeValue =
+ rewriter
+ .create<emitc::CallOp>(
+ /*location=*/loc,
+ /*type=*/emitc::OpaqueType::get(ctx, "iree_vm_type_def_t"),
+ /*callee=*/
+ StringAttr::get(ctx, "iree_vm_make_undefined_type_def"),
+ /*args=*/ArrayAttr{},
+ /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{})
+ .getResult(0);
}
- return cast<emitc::ApplyOp>(elementTypePtr.getDefiningOp());
+ return elementTypeValue;
}
/// Move multiple refs from one set of variables to another set. As these two
@@ -1108,6 +1092,38 @@
/*memberName=*/"allocator",
/*operand=*/module,
/*value=*/allocatorArg);
+ auto &typeTable = typeConverter.typeTable;
+ attachAttribute(moduleOp, "vm.num_types",
+ builder.getI32IntegerAttr(typeTable.size()));
+ if (!typeTable.empty()) {
+ Type typeRefType = emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t");
+ Type typeRefArrayType = emitc::PointerType::get(typeRefType);
+ Value moduleTypes = emitc_builders::structPtrMember(
+ builder, loc, typeRefArrayType, "types", module);
+
+ std::string listType = "!vm.list";
+ for (auto [index, typeDef] : llvm::enumerate(typeTable)) {
+ std::string typeName = typeDef.full_name;
+ typeConverter.mapType(typeDef.type, index);
+ std::string listPrefix = typeName.substr(0, listType.size());
+ if (listType == listPrefix) {
+ typeName = listPrefix;
+ }
+
+ // Remove leading '!' and wrap in quotes
+ if (typeName[0] == '!') {
+ typeName = typeName.substr(1);
+ }
+ typeName = std::string("\"") + typeName + std::string("\"");
+
+ Value stringView =
+ emitc_builders::ireeMakeCstringView(builder, loc, typeName);
+ Value refType = emitc_builders::ireeVmInstanceLookupType(
+ builder, loc, instanceArg, stringView);
+ emitc_builders::arrayElementAssign(builder, loc, moduleTypes, index,
+ refType);
+ }
+ }
Value vmModule = emitc_builders::allocateVariable(
builder, loc, emitc::OpaqueType::get(ctx, "iree_vm_module_t"));
@@ -2795,7 +2811,7 @@
auto typeIdOp = rewriter.create<emitc::CallOp>(
/*location=*/loc,
/*type=*/emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t"),
- /*callee=*/StringAttr::get(ctx, "iree_vm_buffer_type_id"),
+ /*callee=*/StringAttr::get(ctx, "iree_vm_buffer_type"),
/*args=*/ArrayAttr{},
/*templateArgs=*/ArrayAttr{},
/*operands=*/ArrayRef<Value>{});
@@ -3184,24 +3200,18 @@
releaseRefs(rewriter, loc, funcOp, *typeConverter);
- std::string message = std::string("\"") +
- op.getMessage().value_or("").str() +
- std::string("\"");
+ std::string messageStr = std::string("\"") +
+ op.getMessage().value_or("").str() +
+ std::string("\"");
- auto messageOp = rewriter.create<emitc::CallOp>(
- /*location=*/loc,
- /*type=*/emitc::OpaqueType::get(ctx, "iree_string_view_t"),
- /*callee=*/StringAttr::get(ctx, "iree_make_cstring_view"),
- /*args=*/
- ArrayAttr::get(ctx, {emitc::OpaqueAttr::get(ctx, message)}),
- /*templateArgs=*/ArrayAttr{},
- /*operands=*/ArrayRef<Value>{});
+ Value message =
+ emitc_builders::ireeMakeCstringView(rewriter, loc, messageStr);
auto messageSizeOp = emitc_builders::structMember(
rewriter, loc,
/*type=*/emitc::OpaqueType::get(ctx, "iree_host_size_t"),
/*memberName=*/"size",
- /*operand=*/messageOp.getResult(0));
+ /*operand=*/message);
auto messageSizeIntOp = rewriter.create<emitc::CastOp>(
/*location=*/loc,
@@ -3213,7 +3223,7 @@
/*type=*/
emitc::PointerType::get(emitc::OpaqueType::get(ctx, "const char")),
/*memberName=*/"data",
- /*operand=*/messageOp.getResult(0));
+ /*operand=*/message);
auto status = rewriter.create<emitc::CallOp>(
/*location=*/loc,
@@ -3368,19 +3378,29 @@
/*index=*/rewriter.getUI32IntegerAttr(globalOrdinal),
/*operand=*/refs);
+ auto moduleOp = op->getParentOfType<IREE::VM::ModuleOp>();
+ auto parentFuncOp = op->getParentOfType<mlir::func::FuncOp>();
+ const BlockArgument moduleArg =
+ parentFuncOp.getArgument(CCONV_ARGUMENT_MODULE);
Type elementType = localValue.getType();
- auto elementTypePtrOp = createVmTypeDefPtr(rewriter, op, elementType);
+ auto elementTypePtr =
+ createVmTypeDefPtr(rewriter, op->getLoc(), *typeConverter, moduleOp,
+ moduleArg, elementType);
- if (!elementTypePtrOp.has_value()) {
+ if (!elementTypePtr.has_value()) {
return op->emitError() << "generating iree_vm_type_def_t* failed";
}
- auto typedefRefType = emitc_builders::structPtrMember(
- rewriter, loc,
- /*type=*/emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t"),
- /*memberName=*/"ref_type",
- /*operand=*/elementTypePtrOp.value().getResult());
+ auto typedefAsRef =
+ rewriter
+ .create<emitc::CallOp>(
+ /*location=*/loc,
+ /*type=*/emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t"),
+ /*callee=*/StringAttr::get(ctx, "iree_vm_type_def_as_ref"),
+ /*args=*/ArrayAttr{}, /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{elementTypePtr.value()})
+ .getResult(0);
Value srcRef = isLoad ? stateRef : localRef.value();
Value destRef = isLoad ? localRef.value() : stateRef;
@@ -3395,7 +3415,7 @@
ArrayAttr::get(ctx,
{rewriter.getBoolAttr(move), rewriter.getIndexAttr(0),
rewriter.getIndexAttr(1), rewriter.getIndexAttr(2)}),
- /*operands=*/ArrayRef<Value>{srcRef, typedefRefType, destRef},
+ /*operands=*/ArrayRef<Value>{srcRef, typedefAsRef, destRef},
/*typeConverter=*/*typeConverter);
if (isLoad) {
@@ -3727,13 +3747,13 @@
std::optional<CNames> extractCNames(SrcOpTy op) const {
if (isa<IREE::VM::ListAllocOp>(op)) {
- return CNames{"iree_vm_list_t", "iree_vm_list_type_id",
+ return CNames{"iree_vm_list_t", "iree_vm_list_type",
"iree_vm_list_create"};
} else if (isa<IREE::VM::BufferAllocOp>(op)) {
- return CNames{"iree_vm_buffer_t", "iree_vm_buffer_type_id",
+ return CNames{"iree_vm_buffer_t", "iree_vm_buffer_type",
"iree_vm_buffer_create"};
} else if (isa<IREE::VM::BufferCloneOp>(op)) {
- return CNames{"iree_vm_buffer_t", "iree_vm_buffer_type_id",
+ return CNames{"iree_vm_buffer_t", "iree_vm_buffer_type",
"iree_vm_buffer_clone"};
}
return std::nullopt;
@@ -3745,16 +3765,24 @@
Value allocator) const {
SmallVector<Value> result;
- std::optional<emitc::ApplyOp> elementTypePtrOp =
- createVmTypeDefPtr(rewriter, op.getOperation(), elementType);
+ IREE::VM::EmitCTypeConverter *typeConverter =
+ this->template getTypeConverter<IREE::VM::EmitCTypeConverter>();
+ auto moduleOp = op.getOperation()->getParentOfType<IREE::VM::ModuleOp>();
+ auto parentFuncOp =
+ op.getOperation()->getParentOfType<mlir::func::FuncOp>();
+ const BlockArgument moduleArg =
+ parentFuncOp.getArgument(CCONV_ARGUMENT_MODULE);
+ auto elementTypePtr =
+ createVmTypeDefPtr(rewriter, op.getLoc(), *typeConverter, moduleOp,
+ moduleArg, elementType);
- if (!elementTypePtrOp.has_value()) {
+ if (!elementTypePtr.has_value()) {
return std::nullopt;
}
Value capacity = adaptor.getOperands()[0];
- result.push_back(elementTypePtrOp.value().getResult());
+ result.push_back(elementTypePtr.value());
result.push_back(capacity);
result.push_back(allocator);
result.push_back(containerPtr);
@@ -3971,57 +3999,83 @@
ref.value()},
/*typeConverter=*/*typeConverter);
+ auto moduleOp = getOp.getOperation()->getParentOfType<IREE::VM::ModuleOp>();
+ auto parentFuncOp =
+ getOp.getOperation()->getParentOfType<mlir::func::FuncOp>();
+ const BlockArgument moduleArg =
+ parentFuncOp.getArgument(CCONV_ARGUMENT_MODULE);
+
Type elementType = getOp.getResult().getType();
- auto elementTypePtrOp =
- createVmTypeDefPtr(rewriter, getOp.getOperation(), elementType);
+ auto elementTypePtr =
+ createVmTypeDefPtr(rewriter, getOp.getLoc(), *typeConverter, moduleOp,
+ moduleArg, elementType);
- if (!elementTypePtrOp.has_value()) {
+ if (!elementTypePtr.has_value()) {
return getOp.emitError() << "generating iree_vm_type_def_t* failed";
}
// Build the following expression:
// (ref->type != IREE_VM_REF_TYPE_NULL &&
// (iree_vm_type_def_is_value(type_def) || ref->type !=
- // type_def->ref_type))
+ // iree_vm_type_def_as_ref(type_def)))
Value invalidType;
{
+ // ref->type
auto refType = emitc_builders::structPtrMember(
rewriter, loc,
/*type=*/emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t"),
/*memberName=*/"type",
/*operand=*/ref.value());
- auto refTypeNull = rewriter.create<emitc::ConstantOp>(
- /*location=*/loc,
- /*resultType=*/emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t"),
- /*value=*/emitc::OpaqueAttr::get(ctx, "IREE_VM_REF_TYPE_NULL"));
+ // IREE_VM_REF_TYPE_NULL
+ auto refTypeNull =
+ rewriter
+ .create<emitc::ConstantOp>(
+ /*location=*/loc,
+ /*resultType=*/
+ emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t"),
+ /*value=*/
+ emitc::OpaqueAttr::get(ctx, "IREE_VM_REF_TYPE_NULL"))
+ .getResult();
- auto typedefIsValue = rewriter.create<emitc::CallOp>(
- /*location=*/loc,
- /*type=*/rewriter.getI1Type(),
- /*callee=*/StringAttr::get(ctx, "iree_vm_type_def_is_value"),
- /*args=*/ArrayAttr{},
- /*templateArgs=*/ArrayAttr{},
- /*operands=*/
- ArrayRef<Value>{elementTypePtrOp.value().getResult()});
-
- auto typedefRefType = emitc_builders::structPtrMember(
- rewriter, loc,
- /*type=*/emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t"),
- /*memberName=*/"ref_type",
- /*operand=*/elementTypePtrOp.value().getResult());
-
+ // ref->type != IREE_VM_REF_TYPE_NULL
auto refTypeIsNotNull = emitc_builders::binaryOperator(
rewriter, loc, emitc_builders::BinaryOperator::NOT_EQUAL_TO, refType,
- refTypeNull.getResult(), rewriter.getI1Type());
+ refTypeNull, rewriter.getI1Type());
+
+ // (iree_vm_type_def_is_value(type_def)
+ auto typedefIsValue =
+ rewriter
+ .create<emitc::CallOp>(
+ /*location=*/loc,
+ /*type=*/rewriter.getI1Type(),
+ /*callee=*/StringAttr::get(ctx, "iree_vm_type_def_is_value"),
+ /*args=*/ArrayAttr{},
+ /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{elementTypePtr.value()})
+ .getResult(0);
+
+ // iree_vm_type_def_as_ref(type_def)
+ auto typedefAsRef =
+ rewriter
+ .create<emitc::CallOp>(
+ /*location=*/loc,
+ /*type=*/emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t"),
+ /*callee=*/StringAttr::get(ctx, "iree_vm_type_def_as_ref"),
+ /*args=*/ArrayAttr{},
+ /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{elementTypePtr.value()})
+ .getResult(0);
+
+ // ref->type != iree_vm_type_def_as_ref(type_def)
auto refTypesDontMatch = emitc_builders::binaryOperator(
rewriter, loc, emitc_builders::BinaryOperator::NOT_EQUAL_TO, refType,
- typedefRefType, rewriter.getI1Type());
+ typedefAsRef, rewriter.getI1Type());
auto invalidRefType = emitc_builders::binaryOperator(
rewriter, loc, emitc_builders::BinaryOperator::LOGICAL_OR,
- typedefIsValue.getResult(0), refTypesDontMatch, rewriter.getI1Type());
+ typedefIsValue, refTypesDontMatch, rewriter.getI1Type());
invalidType = emitc_builders::binaryOperator(
rewriter, loc, emitc_builders::BinaryOperator::LOGICAL_AND,
@@ -4665,6 +4719,7 @@
ConversionTarget target(getContext());
EmitCTypeConverter typeConverter;
+ typeConverter.cacheTypeTable(module);
// Convert vm.func ops to std.func with the calling convention used by
// EmitC. We convert these upfront to make sure vm.call ops always
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCBuilders.cpp b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCBuilders.cpp
index dc29462..ce8c83e 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCBuilders.cpp
+++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCBuilders.cpp
@@ -203,6 +203,22 @@
ArrayRef<Value>{dest, count});
}
+Value arrayElement(OpBuilder builder, Location location, Type type,
+ size_t index, Value operand) {
+ auto ctx = builder.getContext();
+ return builder
+ .create<emitc::CallOp>(
+ /*location=*/location,
+ /*type=*/type,
+ /*callee=*/StringAttr::get(ctx, "EMITC_ARRAY_ELEMENT"),
+ /*args=*/
+ ArrayAttr::get(
+ ctx, {builder.getIndexAttr(0), builder.getI32IntegerAttr(index)}),
+ /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{operand})
+ .getResult(0);
+}
+
Value arrayElementAddress(OpBuilder builder, Location location, Type type,
IntegerAttr index, Value operand) {
auto ctx = builder.getContext();
@@ -234,6 +250,21 @@
.getResult(0);
}
+void arrayElementAssign(OpBuilder builder, Location location, Value array,
+ size_t index, Value value) {
+ auto ctx = builder.getContext();
+ builder.create<emitc::CallOp>(
+ /*location=*/location,
+ /*type=*/TypeRange{},
+ /*callee=*/StringAttr::get(ctx, "EMITC_ARRAY_ELEMENT_ASSIGN"),
+ /*args=*/
+ ArrayAttr::get(ctx,
+ {builder.getIndexAttr(0), builder.getI32IntegerAttr(index),
+ builder.getIndexAttr(1)}),
+ /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{array, value});
+}
+
void structDefinition(OpBuilder builder, Location location,
StringRef structName, ArrayRef<StructField> fields) {
std::string structBody;
@@ -329,6 +360,21 @@
/*operands=*/ArrayRef<Value>{operand, data});
}
+Value ireeMakeCstringView(OpBuilder builder, Location location,
+ std::string str) {
+ auto ctx = builder.getContext();
+ return builder
+ .create<emitc::CallOp>(
+ /*location=*/location,
+ /*type=*/emitc::OpaqueType::get(ctx, "iree_string_view_t"),
+ /*callee=*/StringAttr::get(ctx, "iree_make_cstring_view"),
+ /*args=*/
+ ArrayAttr::get(ctx, {emitc::OpaqueAttr::get(ctx, str)}),
+ /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{})
+ .getResult(0);
+}
+
Value ireeOkStatus(OpBuilder builder, Location location) {
auto ctx = builder.getContext();
return builder
@@ -342,6 +388,21 @@
.getResult(0);
}
+Value ireeVmInstanceLookupType(OpBuilder builder, Location location,
+ Value instance, Value stringView) {
+ auto ctx = builder.getContext();
+ Type refType = emitc::OpaqueType::get(ctx, "iree_vm_ref_type_t");
+ return builder
+ .create<emitc::CallOp>(
+ /*location=*/location,
+ /*type=*/refType,
+ /*callee=*/StringAttr::get(ctx, "iree_vm_instance_lookup_type"),
+ /*args=*/ArrayAttr{},
+ /*templateArgs=*/ArrayAttr{},
+ /*operands=*/ArrayRef<Value>{instance, stringView})
+ .getResult(0);
+}
+
void ireeVmRefRelease(OpBuilder builder, Location location, Value operand) {
auto ctx = builder.getContext();
builder.create<emitc::CallOp>(
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCBuilders.h b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCBuilders.h
index 98fb58a..fb076bb 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCBuilders.h
+++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCBuilders.h
@@ -82,12 +82,18 @@
void memset(OpBuilder builder, Location location, Value dest, int ch,
Value count);
+Value arrayElement(OpBuilder builder, Location location, Type type,
+ size_t index, Value operand);
+
Value arrayElementAddress(OpBuilder builder, Location location, Type type,
IntegerAttr index, Value operand);
Value arrayElementAddress(OpBuilder builder, Location location, Type type,
Value index, Value operand);
+void arrayElementAssign(OpBuilder builder, Location location, Value array,
+ size_t index, Value value);
+
void structDefinition(OpBuilder builder, Location location,
StringRef structName, ArrayRef<StructField> fields);
@@ -106,8 +112,14 @@
void structPtrMemberAssign(OpBuilder builder, Location location,
StringRef memberName, Value operand, Value data);
+Value ireeMakeCstringView(OpBuilder builder, Location location,
+ std::string str);
+
Value ireeOkStatus(OpBuilder builder, Location location);
+Value ireeVmInstanceLookupType(OpBuilder builder, Location location,
+ Value instance, Value stringView);
+
void ireeVmRefRelease(OpBuilder builder, Location location, Value operand);
} // namespace emitc_builders
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCTypeConverter.h b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCTypeConverter.h
index 2500eb1..7563c08 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCTypeConverter.h
+++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/EmitCTypeConverter.h
@@ -9,6 +9,7 @@
#include "iree/compiler/Dialect/VM/Conversion/VMToEmitC/VMAnalysis.h"
#include "iree/compiler/Dialect/VM/IR/VMTypes.h"
+#include "iree/compiler/Dialect/VM/Utils/TypeTable.h"
#include "mlir/Dialect/EmitC/IR/EmitC.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Transforms/DialectConversion.h"
@@ -37,10 +38,24 @@
Type convertTypeAsPointer(Type type);
emitc::OpaqueType convertTypeAsCType(Type type);
+ void cacheTypeTable(IREE::VM::ModuleOp module) {
+ typeTable = buildTypeTable(module);
+ }
+ void mapType(Type type, size_t index) { typeOrdinalMap[type] = index; }
+ Optional<size_t> lookupType(Type type) {
+ auto ptr = typeOrdinalMap.find(type);
+ if (ptr == typeOrdinalMap.end()) {
+ return std::nullopt;
+ }
+ return ptr->second;
+ }
+
SetVector<Operation *> sourceMaterializations;
VMAnalysisCache analysisCache;
+ std::vector<TypeDef> typeTable;
private:
+ llvm::DenseMap<Type, int> typeOrdinalMap;
FailureOr<std::reference_wrapper<VMAnalysis>> lookupAnalysis(Operation *op);
};
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/buffer_ops.mlir b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/buffer_ops.mlir
index b1407e5..4dd2652 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/buffer_ops.mlir
+++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/buffer_ops.mlir
@@ -11,7 +11,7 @@
// CHECK-DAG: %[[BUFFER_ACCESS:.+]] = "emitc.constant"() {value = #emitc.opaque<"IREE_VM_BUFFER_ACCESS_MUTABLE | IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST">} : () -> !emitc.opaque<"iree_vm_buffer_access_t">
// CHECK-NEXT: %[[STATUS:.+]] = emitc.call "iree_vm_buffer_create"(%[[BUFFER_ACCESS]], %[[SIZE]], %[[ALIGNMENT]], %[[ALLOCTOR]], %[[BUFFER_PTR]]) : (!emitc.opaque<"iree_vm_buffer_access_t">, i64, i32, !emitc.opaque<"iree_allocator_t">, !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>>) -> !emitc.opaque<"iree_status_t">
- // CHECK: %[[BUFFER_TYPE_ID:.+]] = emitc.call "iree_vm_buffer_type_id"() : () -> !emitc.opaque<"iree_vm_ref_type_t">
+ // CHECK: %[[BUFFER_TYPE_ID:.+]] = emitc.call "iree_vm_buffer_type"() : () -> !emitc.opaque<"iree_vm_ref_type_t">
// CHECK-NEXT: %[[STATUS2:.+]] = emitc.call "iree_vm_ref_wrap_assign"(%[[BUFFER]], %[[BUFFER_TYPE_ID]], %1) : (!emitc.ptr<!emitc.opaque<"iree_vm_buffer_t">>, !emitc.opaque<"iree_vm_ref_type_t">, !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_status_t">
%c128 = vm.const.i64 128
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/global_ops.mlir b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/global_ops.mlir
index 6587e82..260f993 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/global_ops.mlir
+++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/global_ops.mlir
@@ -35,11 +35,8 @@
vm.func @global_load_ref() -> !vm.buffer {
// CHECK: %[[A:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%arg2) {args = [0 : index, #emitc.opaque<"refs">]} : (!emitc.ptr<!emitc.opaque<"my_module_state_t">>) -> !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>
// CHECK: %[[B:.+]] = emitc.call "EMITC_ARRAY_ELEMENT_ADDRESS"(%[[A]]) {args = [0 : index, 0 : ui32]} : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>
- // CHECK: %[[C:.+]] = "emitc.variable"() {value = #emitc.opaque<"">} : () -> !emitc.opaque<"iree_vm_type_def_t">
- // CHECK: %[[D:.+]] = emitc.apply "&"(%[[C]]) : (!emitc.opaque<"iree_vm_type_def_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_type_def_t">>
- // CHECK: emitc.call "EMITC_STRUCT_MEMBER_ASSIGN"{{.*}}
- // CHECK: %[[E:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%[[D]]) {args = [0 : index, #emitc.opaque<"ref_type">]} : (!emitc.ptr<!emitc.opaque<"iree_vm_type_def_t">>) -> !emitc.opaque<"iree_vm_ref_type_t">
- // CHECK: %{{.+}} = emitc.call "iree_vm_ref_retain_or_move_checked"(%[[B]], %[[E]], %arg3) {args = [false, 0 : index, 1 : index, 2 : index]} : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>, !emitc.opaque<"iree_vm_ref_type_t">, !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_status_t">
+ // CHECK: %[[C:.+]] = emitc.call "iree_vm_type_def_as_ref"(%{{.+}}) : (!emitc.opaque<"iree_vm_type_def_t">) -> !emitc.opaque<"iree_vm_ref_type_t">
+ // CHECK: %{{.+}} = emitc.call "iree_vm_ref_retain_or_move_checked"(%[[B]], %[[C]], %arg3) {args = [false, 0 : index, 1 : index, 2 : index]} : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>, !emitc.opaque<"iree_vm_ref_type_t">, !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_status_t">
%0 = vm.global.load.ref @g0 : !vm.buffer
vm.return %0 : !vm.buffer
}
@@ -54,11 +51,8 @@
vm.func @global_store_ref(%arg0 : !vm.buffer) {
// CHECK: %[[A:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%arg2) {args = [0 : index, #emitc.opaque<"refs">]} : (!emitc.ptr<!emitc.opaque<"my_module_state_t">>) -> !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>
// CHECK: %[[B:.+]] = emitc.call "EMITC_ARRAY_ELEMENT_ADDRESS"(%[[A]]) {args = [0 : index, 0 : ui32]} : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>
- // CHECK: %[[C:.+]] = "emitc.variable"() {value = #emitc.opaque<"">} : () -> !emitc.opaque<"iree_vm_type_def_t">
- // CHECK: %[[D:.+]] = emitc.apply "&"(%[[C]]) : (!emitc.opaque<"iree_vm_type_def_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_type_def_t">>
- // CHECK: emitc.call "EMITC_STRUCT_MEMBER_ASSIGN"{{.*}}
- // CHECK: %[[E:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%[[D]]) {args = [0 : index, #emitc.opaque<"ref_type">]} : (!emitc.ptr<!emitc.opaque<"iree_vm_type_def_t">>) -> !emitc.opaque<"iree_vm_ref_type_t">
- // CHECK: %{{.+}} = emitc.call "iree_vm_ref_retain_or_move_checked"(%arg3, %[[E]], %[[B]]) {args = [false, 0 : index, 1 : index, 2 : index]} : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>, !emitc.opaque<"iree_vm_ref_type_t">, !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_status_t">
+ // CHECK: %[[C:.+]] = emitc.call "iree_vm_type_def_as_ref"(%{{.+}}) : (!emitc.opaque<"iree_vm_type_def_t">) -> !emitc.opaque<"iree_vm_ref_type_t">
+ // CHECK: %{{.+}} = emitc.call "iree_vm_ref_retain_or_move_checked"(%arg3, %[[C]], %[[B]]) {args = [false, 0 : index, 1 : index, 2 : index]} : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>, !emitc.opaque<"iree_vm_ref_type_t">, !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_status_t">
vm.global.store.ref %arg0, @g0_mut : !vm.buffer
vm.return
}
diff --git a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/list_ops.mlir b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/list_ops.mlir
index cf5f761..8561ddc 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/list_ops.mlir
+++ b/compiler/src/iree/compiler/Dialect/VM/Conversion/VMToEmitC/test/list_ops.mlir
@@ -6,16 +6,13 @@
// CHECK: %[[LIST:.+]] = "emitc.variable"() {value = #emitc.opaque<"NULL">} : () -> !emitc.ptr<!emitc.opaque<"iree_vm_list_t">>
// CHECK: %[[LIST_PTR:.+]] = emitc.apply "&"(%3) : (!emitc.ptr<!emitc.opaque<"iree_vm_list_t">>) -> !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_list_t">>>
// CHECK: %[[ALLOCATOR:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%arg2) {args = [0 : index, #emitc.opaque<"allocator">]} : (!emitc.ptr<!emitc.opaque<"my_module_state_t">>) -> !emitc.opaque<"iree_allocator_t">
- // CHECK-NEXT: %[[TYPE_DEF:.+]] = "emitc.variable"() {value = #emitc.opaque<"">} : () -> !emitc.opaque<"iree_vm_type_def_t">
- // CHECK-NEXT: %[[TYPE_DEF_PTR:.+]] = emitc.apply "&"(%[[TYPE_DEF]]) : (!emitc.opaque<"iree_vm_type_def_t">) -> !emitc.ptr<!emitc.opaque<"iree_vm_type_def_t">>
-
- // CHECK: emitc.call "EMITC_STRUCT_MEMBER_ASSIGN"(%[[TYPE_DEF]]) {args = [0 : index, #emitc.opaque<"value_type">, #emitc.opaque<"IREE_VM_VALUE_TYPE_I32">]} : (!emitc.opaque<"iree_vm_type_def_t">) -> ()
- // CHECK-NEXT: emitc.call "EMITC_STRUCT_MEMBER_ASSIGN"(%[[TYPE_DEF]]) {args = [0 : index, #emitc.opaque<"ref_type">, #emitc.opaque<"IREE_VM_REF_TYPE_NULL">]} : (!emitc.opaque<"iree_vm_type_def_t">) -> ()
- // CHECK-NEXT: %[[STATUS:.+]] = emitc.call "iree_vm_list_create"(%[[TYPE_DEF_PTR]], %arg3, %[[ALLOCATOR]], %[[LIST_PTR]]) : (!emitc.ptr<!emitc.opaque<"iree_vm_type_def_t">>, i32, !emitc.opaque<"iree_allocator_t">, !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_list_t">>>) -> !emitc.opaque<"iree_status_t">
-
- // CHECK: %[[LIST_TYPE_ID:.+]] = emitc.call "iree_vm_list_type_id"() : () -> !emitc.opaque<"iree_vm_ref_type_t">
+
+ // CHECK: %[[TYPE_DEF:.+]] = emitc.call "iree_vm_make_value_type_def"() {args = [#emitc.opaque<"IREE_VM_VALUE_TYPE_I32">]} : () -> !emitc.opaque<"iree_vm_type_def_t">
+ // CHECK-NEXT: %[[STATUS:.+]] = emitc.call "iree_vm_list_create"(%[[TYPE_DEF]], %arg3, %[[ALLOCATOR]], %[[LIST_PTR]]) : (!emitc.opaque<"iree_vm_type_def_t">, i32, !emitc.opaque<"iree_allocator_t">, !emitc.ptr<!emitc.ptr<!emitc.opaque<"iree_vm_list_t">>>) -> !emitc.opaque<"iree_status_t">
+
+ // CHECK: %[[LIST_TYPE_ID:.+]] = emitc.call "iree_vm_list_type"() : () -> !emitc.opaque<"iree_vm_ref_type_t">
// CHECK-NEXT: %[[STATUS2:.+]] = emitc.call "iree_vm_ref_wrap_assign"(%[[LIST]], %[[LIST_TYPE_ID]], %arg4) : (!emitc.ptr<!emitc.opaque<"iree_vm_list_t">>, !emitc.opaque<"iree_vm_ref_type_t">, !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_status_t">
-
+
%0 = vm.list.alloc %arg0 : (i32) -> !vm.list<i32>
vm.return %0 : !vm.list<i32>
}
@@ -85,12 +82,12 @@
// CHECK: %{{.+}} = emitc.call "iree_vm_list_get_ref_retain"(%1, %arg4, %arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_list_t">>, i32, !emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_status_t">
// CHECK: %[[A:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%arg3) {args = [0 : index, #emitc.opaque<"type">]} : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> !emitc.opaque<"iree_vm_ref_type_t">
// CHECK: %[[B:.+]] = "emitc.constant"() {value = #emitc.opaque<"IREE_VM_REF_TYPE_NULL">} : () -> !emitc.opaque<"iree_vm_ref_type_t">
- // CHECK: %[[C:.+]] = emitc.call "iree_vm_type_def_is_value"(%{{.+}}) : (!emitc.ptr<!emitc.opaque<"iree_vm_type_def_t">>) -> i1
- // CHECK: %[[D:.+]] = emitc.call "EMITC_STRUCT_PTR_MEMBER"(%{{.+}}) {args = [0 : index, #emitc.opaque<"ref_type">]} : (!emitc.ptr<!emitc.opaque<"iree_vm_type_def_t">>) -> !emitc.opaque<"iree_vm_ref_type_t">
- // CHECK: %[[E:.+]] = emitc.call "EMITC_BINARY"(%[[A]], %[[B]]) {args = [#emitc.opaque<"!=">, 0 : index, 1 : index]} : (!emitc.opaque<"iree_vm_ref_type_t">, !emitc.opaque<"iree_vm_ref_type_t">) -> i1
- // CHECK: %[[F:.+]] = emitc.call "EMITC_BINARY"(%[[A]], %[[D]]) {args = [#emitc.opaque<"!=">, 0 : index, 1 : index]} : (!emitc.opaque<"iree_vm_ref_type_t">, !emitc.opaque<"iree_vm_ref_type_t">) -> i1
- // CHECK: %[[G:.+]] = emitc.call "EMITC_BINARY"(%[[C]], %[[F]]) {args = [#emitc.opaque<"||">, 0 : index, 1 : index]} : (i1, i1) -> i1
- // CHECK: %{{.+}} = emitc.call "EMITC_BINARY"(%[[E]], %[[G]]) {args = [#emitc.opaque<"&&">, 0 : index, 1 : index]} : (i1, i1) -> i1
+ // CHECK: %[[C:.+]] = emitc.call "EMITC_BINARY"(%[[A]], %[[B]]) {args = [#emitc.opaque<"!=">, 0 : index, 1 : index]} : (!emitc.opaque<"iree_vm_ref_type_t">, !emitc.opaque<"iree_vm_ref_type_t">) -> i1
+ // CHECK: %[[D:.+]] = emitc.call "iree_vm_type_def_is_value"(%{{.+}}) : (!emitc.opaque<"iree_vm_type_def_t">) -> i1
+ // CHECK: %[[E:.+]] = emitc.call "iree_vm_type_def_as_ref"(%{{.+}}) : (!emitc.opaque<"iree_vm_type_def_t">) -> !emitc.opaque<"iree_vm_ref_type_t">
+ // CHECK: %[[F:.+]] = emitc.call "EMITC_BINARY"(%[[A]], %[[E]]) {args = [#emitc.opaque<"!=">, 0 : index, 1 : index]} : (!emitc.opaque<"iree_vm_ref_type_t">, !emitc.opaque<"iree_vm_ref_type_t">) -> i1
+ // CHECK: %[[G:.+]] = emitc.call "EMITC_BINARY"(%[[D]], %[[F]]) {args = [#emitc.opaque<"||">, 0 : index, 1 : index]} : (i1, i1) -> i1
+ // CHECK: %{{.+}} = emitc.call "EMITC_BINARY"(%[[C]], %[[G]]) {args = [#emitc.opaque<"&&">, 0 : index, 1 : index]} : (i1, i1) -> i1
// CHECK: cf.cond_br %{{.+}}, ^[[FAIL:.+]], ^[[CONTINUE:.+]]
// CHECK: ^[[FAIL]]:
// CHECK-NEXT: emitc.call "iree_vm_ref_release"(%arg3) : (!emitc.ptr<!emitc.opaque<"iree_vm_ref_t">>) -> ()
diff --git a/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/BUILD.bazel b/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/BUILD.bazel
index 9cf32a1..627589a 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/BUILD.bazel
+++ b/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/BUILD.bazel
@@ -35,6 +35,7 @@
"//compiler/src/iree/compiler/Dialect/VM/IR",
"//compiler/src/iree/compiler/Dialect/VM/Transforms",
"//compiler/src/iree/compiler/Dialect/VM/Utils:CallingConvention",
+ "//compiler/src/iree/compiler/Dialect/VM/Utils:TypeTable",
"//compiler/src/iree/compiler/Utils",
"//runtime/src/iree/schemas:bytecode_module_def_c_fbs",
"@llvm-project//llvm:Support",
diff --git a/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/BytecodeModuleTarget.cpp b/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/BytecodeModuleTarget.cpp
index 487e54a..fe1bbaa 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/BytecodeModuleTarget.cpp
+++ b/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/BytecodeModuleTarget.cpp
@@ -20,6 +20,7 @@
#include "iree/compiler/Dialect/VM/Target/Bytecode/BytecodeEncoder.h"
#include "iree/compiler/Dialect/VM/Transforms/Passes.h"
#include "iree/compiler/Dialect/VM/Utils/CallingConvention.h"
+#include "iree/compiler/Dialect/VM/Utils/TypeTable.h"
#include "iree/compiler/Utils/FlatbufferUtils.h"
#include "iree/compiler/Utils/TracingUtils.h"
#include "iree/schemas/bytecode_module_def_builder.h"
@@ -61,11 +62,6 @@
// data at the risk of tripping the 31-bit FlatBuffer offset values.
static constexpr int kMaxEmbeddedDataSize = 4 * 1024;
-struct TypeDef {
- Type type;
- std::string full_name;
-};
-
// A rodata reference.
// The archive file is empty if the data is to be embedded in the FlatBuffer.
struct RodataRef {
@@ -131,51 +127,6 @@
return flatbuffers_uint8_vec_end(fbb);
}
-// Finds all types in the module and builds a type table mapping the index in
-// the vector to the type represented by the type ordinal.
-static std::vector<TypeDef> buildTypeTable(IREE::VM::ModuleOp moduleOp) {
- llvm::DenseMap<Type, std::string> typeMap;
- std::function<void(Type)> tryInsertType;
- tryInsertType = [&](Type type) {
- if (auto refPtrType = type.dyn_cast<IREE::VM::RefType>()) {
- type = refPtrType.getObjectType();
- }
- if (typeMap.count(type)) return;
- std::string str;
- llvm::raw_string_ostream sstream(str);
- type.print(sstream);
- sstream.flush();
- typeMap.try_emplace(type, str);
- if (auto listType = type.dyn_cast<IREE::VM::ListType>()) {
- assert(listType.getElementType());
- tryInsertType(listType.getElementType());
- }
- };
- for (auto funcOp : moduleOp.getBlock().getOps<IREE::VM::FuncOp>()) {
- funcOp.walk([&](Operation *op) {
- for (auto type : op->getOperandTypes()) tryInsertType(type);
- for (auto type : op->getResultTypes()) tryInsertType(type);
- });
- }
-
- std::vector<TypeDef> table;
- table.reserve(typeMap.size());
- for (const auto &typeString : typeMap) {
- table.push_back(TypeDef{typeString.first, typeString.second});
- }
- llvm::stable_sort(
- table, +[](const TypeDef &lhs, const TypeDef &rhs) {
- // Always sort builtins above custom types.
- if (lhs.full_name[0] != '!' && rhs.full_name[0] == '!') {
- return true;
- } else if (lhs.full_name[0] == '!' && rhs.full_name[0] != '!') {
- return false;
- }
- return lhs.full_name.compare(rhs.full_name) < 0;
- });
- return table;
-}
-
// Canonicalizes the module to its final form prior to emission.
// This verifies that we only have ops we can serialize and performs any of the
// required transformations (such as debug op stripping).
diff --git a/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/CMakeLists.txt b/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/CMakeLists.txt
index 4c677bd..e0a587b 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Dialect/VM/Target/Bytecode/CMakeLists.txt
@@ -38,6 +38,7 @@
iree::compiler::Dialect::VM::IR
iree::compiler::Dialect::VM::Transforms
iree::compiler::Dialect::VM::Utils::CallingConvention
+ iree::compiler::Dialect::VM::Utils::TypeTable
iree::compiler::Utils
iree::schemas::bytecode_module_def_c_fbs
PUBLIC
diff --git a/compiler/src/iree/compiler/Dialect/VM/Target/C/CModuleTarget.cpp b/compiler/src/iree/compiler/Dialect/VM/Target/C/CModuleTarget.cpp
index 404de82..39945df 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Target/C/CModuleTarget.cpp
+++ b/compiler/src/iree/compiler/Dialect/VM/Target/C/CModuleTarget.cpp
@@ -104,17 +104,23 @@
llvm::raw_ostream &output = emitter.ostream();
std::string moduleName = moduleOp.getName().str();
- output << "struct " << moduleName << "_t {\n";
- output << "iree_allocator_t allocator;\n";
- output << "};\n";
-
- output << "struct " << moduleName << "_state_t {\n";
-
// Returns |count| or 1 if |count| == 0.
// Some compilers (MSVC) don't support zero-length struct fields on the
// interior of structs (just VLA at the tail).
auto countOrEmpty = [](uint32_t count) { return count ? count : 1; };
+ const int64_t numTypes = moduleOp.getOperation()
+ ->getAttr("vm.num_types")
+ .cast<IntegerAttr>()
+ .getInt();
+
+ output << "struct " << moduleName << "_t {\n";
+ output << "iree_allocator_t allocator;\n";
+ output << "iree_vm_ref_type_t types[" << countOrEmpty(numTypes) << "];\n";
+ output << "};\n";
+
+ output << "struct " << moduleName << "_state_t {\n";
+
auto ordinalCounts = moduleOp.getOrdinalCountsAttr();
output << "iree_allocator_t allocator;\n";
output << "uint8_t rwdata[" << countOrEmpty(ordinalCounts.getGlobalBytes())
diff --git a/compiler/src/iree/compiler/Dialect/VM/Utils/BUILD.bazel b/compiler/src/iree/compiler/Dialect/VM/Utils/BUILD.bazel
index 83140ab..46b5c05 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Utils/BUILD.bazel
+++ b/compiler/src/iree/compiler/Dialect/VM/Utils/BUILD.bazel
@@ -23,3 +23,12 @@
"@llvm-project//mlir:Support",
],
)
+
+iree_compiler_cc_library(
+ name = "TypeTable",
+ srcs = ["TypeTable.cpp"],
+ hdrs = ["TypeTable.h"],
+ deps = [
+ "//compiler/src/iree/compiler/Dialect/VM/IR",
+ ],
+)
diff --git a/compiler/src/iree/compiler/Dialect/VM/Utils/CMakeLists.txt b/compiler/src/iree/compiler/Dialect/VM/Utils/CMakeLists.txt
index 61aabc6..b7ee341 100644
--- a/compiler/src/iree/compiler/Dialect/VM/Utils/CMakeLists.txt
+++ b/compiler/src/iree/compiler/Dialect/VM/Utils/CMakeLists.txt
@@ -25,4 +25,16 @@
PUBLIC
)
+iree_cc_library(
+ NAME
+ TypeTable
+ HDRS
+ "TypeTable.h"
+ SRCS
+ "TypeTable.cpp"
+ DEPS
+ iree::compiler::Dialect::VM::IR
+ PUBLIC
+)
+
### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
diff --git a/compiler/src/iree/compiler/Dialect/VM/Utils/TypeTable.cpp b/compiler/src/iree/compiler/Dialect/VM/Utils/TypeTable.cpp
new file mode 100644
index 0000000..f083556
--- /dev/null
+++ b/compiler/src/iree/compiler/Dialect/VM/Utils/TypeTable.cpp
@@ -0,0 +1,62 @@
+// Copyright 2021 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "iree/compiler/Dialect/VM/Utils/TypeTable.h"
+
+namespace mlir {
+namespace iree_compiler {
+namespace IREE {
+namespace VM {
+
+// Finds all types in the module and builds a type table mapping the index in
+// the vector to the type represented by the type ordinal.
+std::vector<TypeDef> buildTypeTable(IREE::VM::ModuleOp moduleOp) {
+ llvm::DenseMap<Type, std::string> typeMap;
+ std::function<void(Type)> tryInsertType;
+ tryInsertType = [&](Type type) {
+ if (auto refPtrType = type.dyn_cast<IREE::VM::RefType>()) {
+ type = refPtrType.getObjectType();
+ }
+ if (typeMap.count(type)) return;
+ std::string str;
+ llvm::raw_string_ostream sstream(str);
+ type.print(sstream);
+ sstream.flush();
+ typeMap.try_emplace(type, str);
+ if (auto listType = type.dyn_cast<IREE::VM::ListType>()) {
+ assert(listType.getElementType());
+ tryInsertType(listType.getElementType());
+ }
+ };
+ for (auto funcOp : moduleOp.getBlock().getOps<IREE::VM::FuncOp>()) {
+ funcOp.walk([&](Operation *op) {
+ for (auto type : op->getOperandTypes()) tryInsertType(type);
+ for (auto type : op->getResultTypes()) tryInsertType(type);
+ });
+ }
+
+ std::vector<TypeDef> table;
+ table.reserve(typeMap.size());
+ for (const auto &typeString : typeMap) {
+ table.push_back(TypeDef{typeString.first, typeString.second});
+ }
+ llvm::stable_sort(
+ table, +[](const TypeDef &lhs, const TypeDef &rhs) {
+ // Always sort builtins above custom types.
+ if (lhs.full_name[0] != '!' && rhs.full_name[0] == '!') {
+ return true;
+ } else if (lhs.full_name[0] == '!' && rhs.full_name[0] != '!') {
+ return false;
+ }
+ return lhs.full_name.compare(rhs.full_name) < 0;
+ });
+ return table;
+}
+
+} // namespace VM
+} // namespace IREE
+} // namespace iree_compiler
+} // namespace mlir
diff --git a/compiler/src/iree/compiler/Dialect/VM/Utils/TypeTable.h b/compiler/src/iree/compiler/Dialect/VM/Utils/TypeTable.h
new file mode 100644
index 0000000..837289c
--- /dev/null
+++ b/compiler/src/iree/compiler/Dialect/VM/Utils/TypeTable.h
@@ -0,0 +1,34 @@
+// Copyright 2023 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef IREE_COMPILER_DIALECT_VM_UTILS_TYPETABLE_H_
+#define IREE_COMPILER_DIALECT_VM_UTILS_TYPETABLE_H_
+
+#include <string>
+#include <vector>
+
+#include "iree/compiler/Dialect/VM/IR/VMOps.h"
+
+namespace mlir {
+namespace iree_compiler {
+namespace IREE {
+namespace VM {
+
+struct TypeDef {
+ Type type;
+ std::string full_name;
+};
+
+// Finds all types in the module and builds a type table mapping the index in
+// the vector to the type represented by the type ordinal.
+std::vector<TypeDef> buildTypeTable(IREE::VM::ModuleOp moduleOp);
+
+} // namespace VM
+} // namespace IREE
+} // namespace iree_compiler
+} // namespace mlir
+
+#endif // IREE_COMPILER_DIALECT_VM_UTILS_TYPETABLE_H_
diff --git a/runtime/bindings/python/hal.cc b/runtime/bindings/python/hal.cc
index bf43d93..6f51450 100644
--- a/runtime/bindings/python/hal.cc
+++ b/runtime/bindings/python/hal.cc
@@ -721,7 +721,7 @@
.def("__repr__", &HalBuffer::Repr);
auto hal_buffer_view = py::class_<HalBufferView>(m, "HalBufferView");
- VmRef::BindRefProtocol(hal_buffer_view, iree_hal_buffer_view_type_id,
+ VmRef::BindRefProtocol(hal_buffer_view, iree_hal_buffer_view_type,
iree_hal_buffer_view_retain_ref,
iree_hal_buffer_view_deref, iree_hal_buffer_view_isa);
hal_buffer_view.def("map", HalMappedMemory::Create, py::keep_alive<0, 1>())
diff --git a/runtime/bindings/python/vm.cc b/runtime/bindings/python/vm.cc
index 70e5f53..b9c0a1b 100644
--- a/runtime/bindings/python/vm.cc
+++ b/runtime/bindings/python/vm.cc
@@ -59,7 +59,8 @@
IREE_TRACE_SCOPE0("VmInstance::Create");
iree_vm_instance_t* instance = NULL;
- auto status = iree_vm_instance_create(iree_allocator_system(), &instance);
+ auto status = iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance);
CheckApiStatus(status, "Error creating instance");
// The python bindings assume the HAL is always available for use.
@@ -182,7 +183,7 @@
const char* const VmRef::kRefAttr = "__iree_vm_ref__";
const char* const VmRef::kCastAttr = "__iree_vm_cast__";
-const char* const VmRef::kTypeIdAttr = "__iree_vm_type_id__";
+const char* const VmRef::kTypeAttr = "__iree_vm_type__";
py::object VmRef::Deref(py::object ref_object_class, bool optional) {
py::object casted = ref_object_class.attr(kCastAttr)(*this);
@@ -193,9 +194,8 @@
}
bool VmRef::IsInstance(py::object ref_object_class) {
- auto type_id =
- py::cast<iree_vm_ref_type_t>(ref_object_class.attr(kTypeIdAttr)());
- return type_id == ref_.type;
+ auto type = py::cast<iree_vm_ref_type_t>(ref_object_class.attr(kTypeAttr)());
+ return type == ref_.type;
}
std::string VmRef::ToString() {
@@ -257,9 +257,11 @@
iree_vm_variant_t v = iree_vm_variant_empty();
CheckApiStatus(iree_vm_list_get_variant_assign(raw_ptr(), index, &v),
"Could not access list element");
- if (iree_vm_type_def_is_value(&v.type)) {
+ if (iree_vm_variant_is_empty(v)) {
+ return py::none();
+ } else if (iree_vm_variant_is_value(v)) {
// Convert a value type.
- switch (v.type.value_type) {
+ switch (iree_vm_type_def_as_value(v.type)) {
case IREE_VM_VALUE_TYPE_I8:
return py::cast(v.i8);
case IREE_VM_VALUE_TYPE_I16:
@@ -275,8 +277,6 @@
default:
throw RaiseValueError("Unsupported VM value type conversion");
}
- } else if (v.type.ref_type == IREE_VM_REF_TYPE_NULL) {
- return py::none();
} else if (iree_vm_variant_is_ref(v)) {
VmRef ref;
iree_vm_ref_retain(&v.ref, &ref.ref());
@@ -290,10 +290,14 @@
iree_vm_variant_t v = iree_vm_variant_empty();
CheckApiStatus(iree_vm_list_get_variant_assign(raw_ptr(), index, &v),
"Could not access list element");
- if (iree_vm_type_def_is_value(&v.type)) {
+ if (iree_vm_variant_is_empty(v)) {
+ py::dict record;
+ record["type"] = "null";
+ return std::move(record);
+ } else if (iree_vm_variant_is_value(v)) {
// Convert a value type.
py::dict record;
- switch (v.type.value_type) {
+ switch (iree_vm_type_def_as_value(v.type)) {
case IREE_VM_VALUE_TYPE_I8:
record["i8"] = py::cast(v.i8);
break;
@@ -317,11 +321,7 @@
}
record["type"] = py::cast("value");
return std::move(record);
- } else if (v.type.ref_type == IREE_VM_REF_TYPE_NULL) {
- py::dict record;
- record["type"] = "null";
- return std::move(record);
- } else if (iree_vm_type_def_is_ref(&v.type)) {
+ } else if (iree_vm_variant_is_ref(v)) {
// Convert reference type.
if (iree_vm_list_isa(v.ref)) {
py::dict record;
@@ -442,7 +442,7 @@
if (iree_vm_variant_is_value(variant)) {
// Convert a value type to a string.
- switch (variant.type.value_type) {
+ switch (iree_vm_type_def_as_value(variant.type)) {
case IREE_VM_VALUE_TYPE_I8: {
out += std::to_string(variant.i8);
break;
@@ -501,7 +501,8 @@
}
out.append("]");
} else {
- out += "Unknown(" + std::to_string(variant.type.ref_type) + ")";
+ out += "Unknown(" +
+ std::to_string(iree_vm_type_def_as_ref(variant.type)) + ")";
}
} else {
out.append("None");
@@ -534,7 +535,7 @@
.export_values();
auto vm_buffer = py::class_<VmBuffer>(m, "VmBuffer", py::buffer_protocol());
- VmRef::BindRefProtocol(vm_buffer, iree_vm_buffer_type_id,
+ VmRef::BindRefProtocol(vm_buffer, iree_vm_buffer_type,
iree_vm_buffer_retain_ref, iree_vm_buffer_deref,
iree_vm_buffer_isa);
vm_buffer
@@ -574,7 +575,7 @@
// Mutation and inspection of the variant list is mostly opaque to python.
auto vm_list = py::class_<VmVariantList>(m, "VmVariantList");
- VmRef::BindRefProtocol(vm_list, iree_vm_list_type_id, iree_vm_list_retain_ref,
+ VmRef::BindRefProtocol(vm_list, iree_vm_list_type, iree_vm_list_retain_ref,
iree_vm_list_deref, iree_vm_list_isa);
vm_list
// User Methods.
diff --git a/runtime/bindings/python/vm.h b/runtime/bindings/python/vm.h
index 464db60..ed3661e 100644
--- a/runtime/bindings/python/vm.h
+++ b/runtime/bindings/python/vm.h
@@ -87,9 +87,10 @@
public:
static VmVariantList Create(iree_host_size_t capacity) {
iree_vm_list_t* list;
- CheckApiStatus(iree_vm_list_create(/*element_type=*/nullptr, capacity,
- iree_allocator_system(), &list),
- "Error allocating variant list");
+ CheckApiStatus(
+ iree_vm_list_create(iree_vm_make_undefined_type_def(), capacity,
+ iree_allocator_system(), &list),
+ "Error allocating variant list");
return VmVariantList::StealFromRawPtr(list);
}
@@ -182,8 +183,8 @@
//----------------------------------------------------------------------------
// Binds the reference protocol to a VmRefObject bound class.
// This defines three attributes:
- // __iree_vm_type_id__()
- // Gets the type id from the object.
+ // __iree_vm_type__()
+ // Gets the type from the object.
// [readonly property] __iree_vm_ref__ :
// Gets a VmRef from the object.
// __iree_vm_cast__(ref) :
@@ -200,13 +201,13 @@
// reference object. It takes some of the C helper functions that are defined
// for each type and is generic.
//----------------------------------------------------------------------------
- static const char* const kTypeIdAttr;
+ static const char* const kTypeAttr;
static const char* const kRefAttr;
static const char* const kCastAttr;
- template <typename PyClass, typename TypeIdFunctor, typename RetainRefFunctor,
+ template <typename PyClass, typename TypeFunctor, typename RetainRefFunctor,
typename DerefFunctor, typename IsaFunctor>
- static void BindRefProtocol(PyClass& cls, TypeIdFunctor type_id,
+ static void BindRefProtocol(PyClass& cls, TypeFunctor type,
RetainRefFunctor retain_ref, DerefFunctor deref,
IsaFunctor isa) {
using WrapperType = typename PyClass::type;
@@ -214,7 +215,7 @@
auto ref_lambda = [=](WrapperType& self) {
return VmRef::Steal(retain_ref(self.raw_ptr()));
};
- cls.def_static(VmRef::kTypeIdAttr, [=]() { return type_id(); });
+ cls.def_static(VmRef::kTypeAttr, [=]() { return type(); });
cls.def_property_readonly(VmRef::kRefAttr, ref_lambda);
cls.def_property_readonly("ref", ref_lambda);
cls.def_static(VmRef::kCastAttr, [=](VmRef& ref) -> py::object {
diff --git a/runtime/bindings/tflite/interpreter.c b/runtime/bindings/tflite/interpreter.c
index 0aaeb92..b7c7f53 100644
--- a/runtime/bindings/tflite/interpreter.c
+++ b/runtime/bindings/tflite/interpreter.c
@@ -91,7 +91,7 @@
_TfLiteInterpreterShapeFrame* frame) {
// [int32...] storage for the shape dimension inputs/outputs.
iree_vm_type_def_t dim_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
IREE_RETURN_IF_ERROR(iree_vm_list_initialize(
iree_make_byte_span(frame->shape_list_storage,
IREE_ARRAYSIZE(frame->shape_list_storage)),
@@ -107,7 +107,7 @@
// Arg 1 is always the shape list for all I/O, so do that once here.
iree_vm_ref_t shape_list_ref = {0};
IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_assign(
- frame->shape_list, iree_vm_list_type_id(), &shape_list_ref));
+ frame->shape_list, iree_vm_list_type(), &shape_list_ref));
IREE_RETURN_IF_ERROR(
iree_vm_list_set_ref_retain(frame->arg_list, 1, &shape_list_ref));
@@ -233,7 +233,7 @@
iree_host_align(sizeof(TfLiteInterpreter), iree_max_align_t);
iree_vm_type_def_t buffer_view_type_def =
- iree_vm_type_def_make_ref_type(iree_hal_buffer_type_id());
+ iree_vm_make_ref_type_def(iree_hal_buffer_type());
total_size +=
iree_vm_list_storage_size(&buffer_view_type_def, model->input_count);
total_size +=
@@ -264,7 +264,7 @@
iree_host_align(sizeof(*interpreter), iree_max_align_t);
iree_vm_type_def_t buffer_view_type_def =
- iree_vm_type_def_make_ref_type(iree_hal_buffer_type_id());
+ iree_vm_make_ref_type_def(iree_hal_buffer_type());
iree_byte_span_t input_list_storage = iree_make_byte_span(
p, iree_vm_list_storage_size(&buffer_view_type_def, model->input_count));
@@ -588,8 +588,8 @@
// NOTE: we could defer the mapping unless requested and ensure state buffers
// remain where they currently are for the next invocation.
for (iree_host_size_t i = 0; i < interpreter->model->output_count; ++i) {
- iree_hal_buffer_t* buffer = (iree_hal_buffer_t*)iree_vm_list_get_ref_deref(
- interpreter->output_list, i, &iree_hal_buffer_descriptor);
+ iree_hal_buffer_t* buffer =
+ iree_vm_list_get_buffer_assign(interpreter->output_list, i);
TfLiteTensor* tensor = &interpreter->output_tensors[i];
IREE_RETURN_IF_ERROR(_TfLiteTensorBind(tensor, buffer));
}
diff --git a/runtime/bindings/tflite/model.c b/runtime/bindings/tflite/model.c
index 55773a0..a3becd7 100644
--- a/runtime/bindings/tflite/model.c
+++ b/runtime/bindings/tflite/model.c
@@ -34,7 +34,8 @@
IREE_TRACE_ZONE_BEGIN(z0);
IREE_RETURN_AND_END_ZONE_IF_ERROR(
- z0, iree_vm_instance_create(allocator, &model->instance));
+ z0, iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT, allocator,
+ &model->instance));
IREE_RETURN_AND_END_ZONE_IF_ERROR(
z0, iree_hal_module_register_all_types(model->instance));
diff --git a/runtime/src/iree/hal/driver_registry.c b/runtime/src/iree/hal/driver_registry.c
index e5e1972..7f45f45 100644
--- a/runtime/src/iree/hal/driver_registry.c
+++ b/runtime/src/iree/hal/driver_registry.c
@@ -148,7 +148,6 @@
if (iree_status_is_ok(status)) {
// Compact list. Note that registration order is preserved.
- // C4090 bug in MSVC: https://tinyurl.com/y46hlogx
memmove((void*)®istry->factories[index], ®istry->factories[index + 1],
registry->factory_count - index - 1);
registry->factories[--registry->factory_count] = NULL;
diff --git a/runtime/src/iree/hal/local/loaders/vmvx_module_loader.c b/runtime/src/iree/hal/local/loaders/vmvx_module_loader.c
index 5ed04e3..40d60f9 100644
--- a/runtime/src/iree/hal/local/loaders/vmvx_module_loader.c
+++ b/runtime/src/iree/hal/local/loaders/vmvx_module_loader.c
@@ -77,7 +77,7 @@
uint8_t input_storage[64] = {0};
iree_vm_list_t* inputs = NULL;
iree_vm_type_def_t element_type =
- iree_vm_type_def_make_ref_type(iree_vm_buffer_type_id());
+ iree_vm_make_ref_type_def(iree_vm_buffer_type());
status = iree_vm_list_initialize(
iree_make_byte_span(input_storage, sizeof(input_storage)), &element_type,
1, &inputs);
@@ -388,7 +388,7 @@
// The list would only need to be constructed once and we could avoid the
// extraneous retain/releases and mappings.
iree_vm_type_def_t buffer_type =
- iree_vm_type_def_make_ref_type(iree_vm_buffer_type_id());
+ iree_vm_make_ref_type_def(iree_vm_buffer_type());
iree_host_size_t binding_list_size =
iree_vm_list_storage_size(&buffer_type, dispatch_state->binding_count);
void* binding_list_storage = iree_alloca(binding_list_size);
@@ -415,7 +415,7 @@
iree_allocator_null(), binding_buffer);
iree_vm_ref_t ref = {0};
status =
- iree_vm_ref_wrap_assign(binding_buffer, iree_vm_buffer_type_id(), &ref);
+ iree_vm_ref_wrap_assign(binding_buffer, iree_vm_buffer_type(), &ref);
if (!iree_status_is_ok(status)) break;
status = iree_vm_list_push_ref_retain(binding_list, &ref);
if (!iree_status_is_ok(status)) break;
@@ -478,21 +478,18 @@
} call_args = {
.local_memory =
{
- .type = iree_vm_buffer_type_id(),
+ .type = iree_vm_buffer_type(),
.ptr = &local_memory_buffer,
- .offsetof_counter = 0,
},
.constants =
{
- .type = iree_vm_buffer_type_id(),
+ .type = iree_vm_buffer_type(),
.ptr = &constants_buffer,
- .offsetof_counter = 0,
},
.bindings =
{
- .type = iree_vm_list_type_id(),
+ .type = iree_vm_list_type(),
.ptr = binding_list,
- .offsetof_counter = 0,
},
.workgroup_id_x = workgroup_state->workgroup_id_x,
.workgroup_id_y = workgroup_state->workgroup_id_y,
@@ -623,7 +620,8 @@
iree_vm_instance_t* instance = NULL;
IREE_RETURN_AND_END_ZONE_IF_ERROR(
- z0, iree_vm_instance_create(host_allocator, &instance));
+ z0, iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT, host_allocator,
+ &instance));
iree_status_t status = iree_hal_vmvx_module_loader_create(
instance, user_module_count, user_modules, host_allocator,
diff --git a/runtime/src/iree/modules/check/check_test.cc b/runtime/src/iree/modules/check/check_test.cc
index 82978ca..acdbdef 100644
--- a/runtime/src/iree/modules/check/check_test.cc
+++ b/runtime/src/iree/modules/check/check_test.cc
@@ -27,8 +27,8 @@
class CheckTest : public ::testing::Test {
protected:
static void SetUpTestSuite() {
- IREE_ASSERT_OK(
- iree_vm_instance_create(iree_allocator_system(), &instance_));
+ IREE_ASSERT_OK(iree_vm_instance_create(
+ IREE_VM_TYPE_CAPACITY_DEFAULT, iree_allocator_system(), &instance_));
IREE_ASSERT_OK(iree_hal_module_register_all_types(instance_));
iree_hal_driver_t* hal_driver = nullptr;
@@ -180,7 +180,7 @@
iree_status_t InvokeValue(const char* function_name,
std::vector<iree_vm_value_t> args) {
IREE_RETURN_IF_ERROR(
- iree_vm_list_create(/*element_type=*/nullptr, args.size(),
+ iree_vm_list_create(iree_vm_make_undefined_type_def(), args.size(),
iree_allocator_system(), &inputs_));
for (auto& arg : args) {
IREE_RETURN_IF_ERROR(iree_vm_list_push_value(inputs_.get(), &arg));
@@ -191,7 +191,7 @@
iree_status_t Invoke(const char* function_name,
std::vector<vm::ref<iree_hal_buffer_view_t>> args) {
IREE_RETURN_IF_ERROR(
- iree_vm_list_create(/*element_type=*/nullptr, args.size(),
+ iree_vm_list_create(iree_vm_make_undefined_type_def(), args.size(),
iree_allocator_system(), &inputs_));
for (auto& arg : args) {
iree_vm_ref_t arg_ref = iree_hal_buffer_view_move_ref(arg.get());
diff --git a/runtime/src/iree/modules/hal/types.c b/runtime/src/iree/modules/hal/types.c
index ba24f9e..4611126 100644
--- a/runtime/src/iree/modules/hal/types.c
+++ b/runtime/src/iree/modules/hal/types.c
@@ -30,38 +30,33 @@
// Type registration
//===----------------------------------------------------------------------===//
-#define IREE_VM_REGISTER_HAL_C_TYPE(type, name, destroy_fn, descriptor) \
- descriptor.type_name = iree_make_cstring_view(name); \
- descriptor.offsetof_counter = offsetof(iree_hal_resource_t, ref_count); \
- descriptor.destroy = (iree_vm_ref_destroy_t)destroy_fn; \
- IREE_RETURN_IF_ERROR(iree_vm_ref_register_type(&descriptor));
+#define IREE_VM_REGISTER_HAL_C_TYPE(instance, type, name, destroy_fn, \
+ registration) \
+ static const iree_vm_ref_type_descriptor_t registration##_storage = { \
+ .type_name = IREE_SVL(name), \
+ .offsetof_counter = offsetof(iree_hal_resource_t, ref_count) / \
+ IREE_VM_REF_COUNTER_ALIGNMENT, \
+ .destroy = (iree_vm_ref_destroy_t)destroy_fn, \
+ }; \
+ IREE_RETURN_IF_ERROR(iree_vm_instance_register_type( \
+ instance, ®istration##_storage, ®istration));
static iree_status_t iree_hal_module_register_common_types(
iree_vm_instance_t* instance) {
- static bool has_registered = false;
- if (has_registered) return iree_ok_status();
-
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_buffer_t, "hal.buffer",
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_buffer_t, "hal.buffer",
iree_hal_buffer_recycle,
- iree_hal_buffer_descriptor);
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_buffer_view_t, "hal.buffer_view",
- iree_hal_buffer_view_destroy,
- iree_hal_buffer_view_descriptor);
-
- has_registered = true;
+ iree_hal_buffer_registration);
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_buffer_view_t,
+ "hal.buffer_view", iree_hal_buffer_view_destroy,
+ iree_hal_buffer_view_registration);
return iree_ok_status();
}
static iree_status_t iree_hal_module_register_executable_types(
iree_vm_instance_t* instance) {
- static bool has_registered = false;
- if (has_registered) return iree_ok_status();
-
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_executable_t, "hal.executable",
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_executable_t, "hal.executable",
iree_hal_executable_destroy,
- iree_hal_executable_descriptor);
-
- has_registered = true;
+ iree_hal_executable_registration);
return iree_ok_status();
}
@@ -79,42 +74,106 @@
IREE_API_EXPORT iree_status_t
iree_hal_module_register_all_types(iree_vm_instance_t* instance) {
- static bool has_registered = false;
- if (has_registered) return iree_ok_status();
-
IREE_RETURN_IF_ERROR(iree_hal_module_register_common_types(instance));
IREE_RETURN_IF_ERROR(iree_hal_module_register_executable_types(instance));
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_allocator_t, "hal.allocator",
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_allocator_t, "hal.allocator",
iree_hal_allocator_destroy,
- iree_hal_allocator_descriptor);
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_channel_t, "hal.channel",
+ iree_hal_allocator_registration);
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_channel_t, "hal.channel",
iree_hal_channel_destroy,
- iree_hal_channel_descriptor);
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_command_buffer_t, "hal.command_buffer",
- iree_hal_command_buffer_destroy,
- iree_hal_command_buffer_descriptor);
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_descriptor_set_layout_t,
+ iree_hal_channel_registration);
+ IREE_VM_REGISTER_HAL_C_TYPE(
+ instance, iree_hal_command_buffer_t, "hal.command_buffer",
+ iree_hal_command_buffer_destroy, iree_hal_command_buffer_registration);
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_descriptor_set_layout_t,
"hal.descriptor_set_layout",
iree_hal_descriptor_set_layout_destroy,
- iree_hal_descriptor_set_layout_descriptor);
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_device_t, "hal.device",
+ iree_hal_descriptor_set_layout_registration);
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_device_t, "hal.device",
iree_hal_device_destroy,
- iree_hal_device_descriptor);
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_event_t, "hal.event",
+ iree_hal_device_registration);
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_event_t, "hal.event",
iree_hal_event_destroy,
- iree_hal_event_descriptor);
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_fence_t, "hal.fence",
+ iree_hal_event_registration);
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_fence_t, "hal.fence",
iree_hal_fence_destroy,
- iree_hal_fence_descriptor);
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_pipeline_layout_t, "hal.pipeline_layout",
- iree_hal_pipeline_layout_destroy,
- iree_hal_pipeline_layout_descriptor);
- IREE_VM_REGISTER_HAL_C_TYPE(iree_hal_semaphore_t, "hal.semaphore",
+ iree_hal_fence_registration);
+ IREE_VM_REGISTER_HAL_C_TYPE(
+ instance, iree_hal_pipeline_layout_t, "hal.pipeline_layout",
+ iree_hal_pipeline_layout_destroy, iree_hal_pipeline_layout_registration);
+ IREE_VM_REGISTER_HAL_C_TYPE(instance, iree_hal_semaphore_t, "hal.semaphore",
iree_hal_semaphore_destroy,
- iree_hal_semaphore_descriptor);
+ iree_hal_semaphore_registration);
- has_registered = true;
+ return iree_ok_status();
+}
+
+#define IREE_VM_RESOLVE_HAL_C_TYPE(instance, type, name, registration) \
+ registration = \
+ iree_vm_instance_lookup_type(instance, iree_make_cstring_view(name)); \
+ if (!registration) { \
+ return iree_make_status(IREE_STATUS_INTERNAL, \
+ "VM type `" name \
+ "` not registered with the instance"); \
+ }
+
+static iree_status_t iree_hal_module_resolve_common_types(
+ iree_vm_instance_t* instance) {
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_buffer_t, "hal.buffer",
+ iree_hal_buffer_registration);
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_buffer_view_t,
+ "hal.buffer_view",
+ iree_hal_buffer_view_registration);
+ return iree_ok_status();
+}
+
+static iree_status_t iree_hal_module_resolve_executable_types(
+ iree_vm_instance_t* instance) {
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_executable_t, "hal.executable",
+ iree_hal_executable_registration);
+ return iree_ok_status();
+}
+
+IREE_API_EXPORT iree_status_t
+iree_hal_module_resolve_inline_types(iree_vm_instance_t* instance) {
+ return iree_hal_module_resolve_common_types(instance);
+}
+
+IREE_API_EXPORT iree_status_t
+iree_hal_module_resolve_loader_types(iree_vm_instance_t* instance) {
+ IREE_RETURN_IF_ERROR(iree_hal_module_resolve_common_types(instance));
+ IREE_RETURN_IF_ERROR(iree_hal_module_resolve_executable_types(instance));
+ return iree_ok_status();
+}
+
+IREE_API_EXPORT iree_status_t
+iree_hal_module_resolve_all_types(iree_vm_instance_t* instance) {
+ IREE_RETURN_IF_ERROR(iree_hal_module_resolve_common_types(instance));
+ IREE_RETURN_IF_ERROR(iree_hal_module_resolve_executable_types(instance));
+
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_allocator_t, "hal.allocator",
+ iree_hal_allocator_registration);
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_channel_t, "hal.channel",
+ iree_hal_channel_registration);
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_command_buffer_t,
+ "hal.command_buffer",
+ iree_hal_command_buffer_registration);
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_descriptor_set_layout_t,
+ "hal.descriptor_set_layout",
+ iree_hal_descriptor_set_layout_registration);
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_device_t, "hal.device",
+ iree_hal_device_registration);
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_event_t, "hal.event",
+ iree_hal_event_registration);
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_fence_t, "hal.fence",
+ iree_hal_fence_registration);
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_pipeline_layout_t,
+ "hal.pipeline_layout",
+ iree_hal_pipeline_layout_registration);
+ IREE_VM_RESOLVE_HAL_C_TYPE(instance, iree_hal_semaphore_t, "hal.semaphore",
+ iree_hal_semaphore_registration);
+
return iree_ok_status();
}
@@ -122,10 +181,31 @@
// Utilities
//===--------------------------------------------------------------------===//
+IREE_API_EXPORT iree_hal_buffer_t* iree_vm_list_get_buffer_assign(
+ const iree_vm_list_t* list, iree_host_size_t i) {
+ return (iree_hal_buffer_t*)iree_vm_list_get_ref_deref(list, i,
+ iree_hal_buffer_type());
+}
+
+IREE_API_EXPORT iree_hal_buffer_t* iree_vm_list_get_buffer_retain(
+ const iree_vm_list_t* list, iree_host_size_t i) {
+ iree_hal_buffer_t* value = iree_vm_list_get_buffer_assign(list, i);
+ iree_hal_buffer_retain(value);
+ return value;
+}
+
+IREE_API_EXPORT iree_status_t iree_vm_list_set_buffer_retain(
+ iree_vm_list_t* list, iree_host_size_t i, iree_hal_buffer_t* value) {
+ iree_vm_ref_t value_ref;
+ IREE_RETURN_IF_ERROR(
+ iree_vm_ref_wrap_assign(value, iree_hal_buffer_type(), &value_ref));
+ return iree_vm_list_set_ref_retain(list, i, &value_ref);
+}
+
IREE_API_EXPORT iree_hal_buffer_view_t* iree_vm_list_get_buffer_view_assign(
const iree_vm_list_t* list, iree_host_size_t i) {
return (iree_hal_buffer_view_t*)iree_vm_list_get_ref_deref(
- list, i, &iree_hal_buffer_view_descriptor);
+ list, i, iree_hal_buffer_view_type());
}
IREE_API_EXPORT iree_hal_buffer_view_t* iree_vm_list_get_buffer_view_retain(
@@ -138,7 +218,7 @@
IREE_API_EXPORT iree_status_t iree_vm_list_set_buffer_view_retain(
iree_vm_list_t* list, iree_host_size_t i, iree_hal_buffer_view_t* value) {
iree_vm_ref_t value_ref;
- IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_assign(
- value, iree_hal_buffer_view_type_id(), &value_ref));
+ IREE_RETURN_IF_ERROR(
+ iree_vm_ref_wrap_assign(value, iree_hal_buffer_view_type(), &value_ref));
return iree_vm_list_set_ref_retain(list, i, &value_ref);
}
diff --git a/runtime/src/iree/modules/hal/types.h b/runtime/src/iree/modules/hal/types.h
index 6036c2c..46f4006 100644
--- a/runtime/src/iree/modules/hal/types.h
+++ b/runtime/src/iree/modules/hal/types.h
@@ -36,28 +36,46 @@
#endif // __cplusplus
// Registers the custom types used by the inline HAL module.
-// WARNING: not thread-safe; call at startup before using.
IREE_API_EXPORT iree_status_t
iree_hal_module_register_inline_types(iree_vm_instance_t* instance);
// Registers the custom types used by the dynamic HAL executable loader module.
-// WARNING: not thread-safe; call at startup before using.
IREE_API_EXPORT iree_status_t
iree_hal_module_register_loader_types(iree_vm_instance_t* instance);
// Registers the custom types used by the full HAL module.
-// WARNING: not thread-safe; call at startup before using.
+// This should only be called in the hosting executable/library that has the
+// IREE VM/HAL compiled in.
IREE_API_EXPORT iree_status_t
iree_hal_module_register_all_types(iree_vm_instance_t* instance);
+// Resolves the custom types used by the inline HAL module.
+IREE_API_EXPORT iree_status_t
+iree_hal_module_resolve_inline_types(iree_vm_instance_t* instance);
+
+// Resolves the custom types used by the dynamic HAL executable loader module.
+IREE_API_EXPORT iree_status_t
+iree_hal_module_resolve_loader_types(iree_vm_instance_t* instance);
+
+// Resolves all HAL types by looking them up on the instance.
+// This should only be called in dynamically-loaded libraries that contain only
+// the HAL shims.
+IREE_API_EXPORT iree_status_t
+iree_hal_module_resolve_all_types(iree_vm_instance_t* instance);
+
// TODO(benvanik): generate these list helpers:
+IREE_API_EXPORT iree_hal_buffer_t* iree_vm_list_get_buffer_assign(
+ const iree_vm_list_t* list, iree_host_size_t i);
+IREE_API_EXPORT iree_hal_buffer_t* iree_vm_list_get_buffer_retain(
+ const iree_vm_list_t* list, iree_host_size_t i);
+IREE_API_EXPORT iree_status_t iree_vm_list_set_buffer_retain(
+ iree_vm_list_t* list, iree_host_size_t i, iree_hal_buffer_t* value);
+
IREE_API_EXPORT iree_hal_buffer_view_t* iree_vm_list_get_buffer_view_assign(
const iree_vm_list_t* list, iree_host_size_t i);
-
IREE_API_EXPORT iree_hal_buffer_view_t* iree_vm_list_get_buffer_view_retain(
const iree_vm_list_t* list, iree_host_size_t i);
-
IREE_API_EXPORT iree_status_t iree_vm_list_set_buffer_view_retain(
iree_vm_list_t* list, iree_host_size_t i, iree_hal_buffer_view_t* value);
diff --git a/runtime/src/iree/modules/hal/utils/buffer_diagnostics.c b/runtime/src/iree/modules/hal/utils/buffer_diagnostics.c
index 745332c..427df99 100644
--- a/runtime/src/iree/modules/hal/utils/buffer_diagnostics.c
+++ b/runtime/src/iree/modules/hal/utils/buffer_diagnostics.c
@@ -22,8 +22,8 @@
IREE_RETURN_IF_ERROR(iree_hal_buffer_check_deref(buffer_ref, &buffer));
iree_vm_buffer_t* message = NULL;
IREE_RETURN_IF_ERROR(iree_vm_buffer_check_deref(message_ref, &message));
- iree_string_view_t message_str IREE_ATTRIBUTE_UNUSED =
- iree_vm_buffer_as_string(message);
+ iree_string_view_t message_str = iree_vm_buffer_as_string(message);
+ (void)message_str;
// Ensure we have enough bytes in the buffer for the encoding we have.
// Note that having more bytes is fine:
@@ -145,8 +145,8 @@
iree_hal_buffer_view_check_deref(buffer_view_ref, &buffer_view));
iree_vm_buffer_t* message = NULL;
IREE_RETURN_IF_ERROR(iree_vm_buffer_check_deref(message_ref, &message));
- iree_string_view_t message_str IREE_ATTRIBUTE_UNUSED =
- iree_vm_buffer_as_string(message);
+ iree_string_view_t message_str = iree_vm_buffer_as_string(message);
+ (void)message_str;
// Check encoding first; getting the encoding wrong is worse than the shape.
// If the actual encoding is opaque we allow it to pass through - this lets
diff --git a/runtime/src/iree/runtime/call.c b/runtime/src/iree/runtime/call.c
index 764668a..1230216 100644
--- a/runtime/src/iree/runtime/call.c
+++ b/runtime/src/iree/runtime/call.c
@@ -43,12 +43,13 @@
// this interface a few small pooled malloc calls should be fine.
iree_allocator_t host_allocator =
iree_runtime_session_host_allocator(session);
- iree_status_t status = iree_vm_list_create(
- /*element_type=*/NULL, arguments.size, host_allocator, &out_call->inputs);
+ iree_status_t status =
+ iree_vm_list_create(iree_vm_make_undefined_type_def(), arguments.size,
+ host_allocator, &out_call->inputs);
if (iree_status_is_ok(status)) {
- status = iree_vm_list_create(
- /*element_type=*/NULL, results.size, host_allocator,
- &out_call->outputs);
+ status =
+ iree_vm_list_create(iree_vm_make_undefined_type_def(), results.size,
+ host_allocator, &out_call->outputs);
}
if (!iree_status_is_ok(status)) {
@@ -107,7 +108,7 @@
IREE_ASSERT_ARGUMENT(buffer_view);
iree_vm_ref_t value = {0};
IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_assign(
- buffer_view, iree_hal_buffer_view_type_id(), &value));
+ buffer_view, iree_hal_buffer_view_type(), &value));
return iree_vm_list_push_ref_retain(call->inputs, &value);
}
diff --git a/runtime/src/iree/runtime/instance.c b/runtime/src/iree/runtime/instance.c
index a61a798..78f5e39 100644
--- a/runtime/src/iree/runtime/instance.c
+++ b/runtime/src/iree/runtime/instance.c
@@ -75,8 +75,8 @@
instance->driver_registry = options->driver_registry;
// TODO(benvanik): driver registry ref counting.
- iree_status_t status =
- iree_vm_instance_create(host_allocator, &instance->vm_instance);
+ iree_status_t status = iree_vm_instance_create(
+ IREE_VM_TYPE_CAPACITY_DEFAULT, host_allocator, &instance->vm_instance);
if (iree_status_is_ok(status)) {
status = iree_hal_module_register_all_types(instance->vm_instance);
}
diff --git a/runtime/src/iree/tooling/comparison.cc b/runtime/src/iree/tooling/comparison.cc
index b01e3c9..c1ac9e6 100644
--- a/runtime/src/iree/tooling/comparison.cc
+++ b/runtime/src/iree/tooling/comparison.cc
@@ -42,7 +42,7 @@
return iree_string_builder_append_string(builder, IREE_SV("empty"));
} else if (iree_vm_variant_is_value(variant)) {
const char* type = NULL;
- switch (variant.type.value_type) {
+ switch (iree_vm_type_def_as_value(variant.type)) {
case IREE_VM_VALUE_TYPE_I8:
type = "i8";
break;
@@ -68,7 +68,7 @@
return iree_string_builder_append_cstring(builder, type);
} else if (iree_vm_variant_is_ref(variant)) {
return iree_string_builder_append_string(
- builder, iree_vm_ref_type_name(variant.type.ref_type));
+ builder, iree_vm_ref_type_name(iree_vm_type_def_as_ref(variant.type)));
} else {
return iree_string_builder_append_string(builder, IREE_SV("unknown"));
}
@@ -78,9 +78,9 @@
iree_vm_variant_t expected_variant,
iree_vm_variant_t actual_variant,
iree_string_builder_t* builder) {
- IREE_ASSERT_EQ(expected_variant.type.value_type,
- actual_variant.type.value_type);
- switch (expected_variant.type.value_type) {
+ IREE_ASSERT_TRUE(
+ iree_vm_type_def_equal(expected_variant.type, actual_variant.type));
+ switch (iree_vm_type_def_as_value(expected_variant.type)) {
case IREE_VM_VALUE_TYPE_I8:
if (expected_variant.i8 != actual_variant.i8) {
IREE_CHECK_OK(iree_string_builder_append_format(
@@ -200,7 +200,7 @@
return true; // both empty
} else if (iree_vm_variant_is_value(actual_variant) &&
iree_vm_variant_is_value(expected_variant)) {
- if (expected_variant.type.value_type != actual_variant.type.value_type) {
+ if (!iree_vm_type_def_equal(expected_variant.type, actual_variant.type)) {
return iree_tooling_compare_values(result_index, expected_variant,
actual_variant, builder);
}
diff --git a/runtime/src/iree/tooling/comparison_test.cc b/runtime/src/iree/tooling/comparison_test.cc
index b8e55b8..11e3c14 100644
--- a/runtime/src/iree/tooling/comparison_test.cc
+++ b/runtime/src/iree/tooling/comparison_test.cc
@@ -37,7 +37,8 @@
class ComparisonTest : public ::testing::Test {
protected:
virtual void SetUp() {
- IREE_ASSERT_OK(iree_vm_instance_create(host_allocator_, &instance_));
+ IREE_ASSERT_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ host_allocator_, &instance_));
IREE_ASSERT_OK(iree_hal_module_register_all_types(instance_));
IREE_ASSERT_OK(iree_hal_allocator_create_heap(
IREE_SV("heap"), host_allocator_, host_allocator_, &device_allocator_));
diff --git a/runtime/src/iree/tooling/context_util.c b/runtime/src/iree/tooling/context_util.c
index 9bc9398..8d2b267 100644
--- a/runtime/src/iree/tooling/context_util.c
+++ b/runtime/src/iree/tooling/context_util.c
@@ -424,7 +424,8 @@
iree_vm_instance_t* instance = NULL;
IREE_RETURN_AND_END_ZONE_IF_ERROR(
- z0, iree_vm_instance_create(host_allocator, &instance));
+ z0, iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT, host_allocator,
+ &instance));
// HACK: to load modules we need the types registered even though we don't
// know if the types are used.
diff --git a/runtime/src/iree/tooling/instrument_util.c b/runtime/src/iree/tooling/instrument_util.c
index 5566931..5ff15ee 100644
--- a/runtime/src/iree/tooling/instrument_util.c
+++ b/runtime/src/iree/tooling/instrument_util.c
@@ -73,11 +73,12 @@
// list and use that across all of them.
iree_vm_list_t* iovec_list = NULL;
IREE_RETURN_AND_END_ZONE_IF_ERROR(
- z0, iree_vm_list_create(NULL, 8, host_allocator, &iovec_list));
+ z0, iree_vm_list_create(iree_vm_make_undefined_type_def(), 8,
+ host_allocator, &iovec_list));
iree_vm_list_t* input_list = NULL;
- iree_status_t status =
- iree_vm_list_create(NULL, 8, host_allocator, &input_list);
+ iree_status_t status = iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 8, host_allocator, &input_list);
if (iree_status_is_ok(status)) {
iree_vm_ref_t iovec_list_ref = iree_vm_list_retain_ref(iovec_list);
status = iree_vm_list_push_ref_move(input_list, &iovec_list_ref);
diff --git a/runtime/src/iree/tooling/trace_replay.c b/runtime/src/iree/tooling/trace_replay.c
index 0778e71..211be60 100644
--- a/runtime/src/iree/tooling/trace_replay.c
+++ b/runtime/src/iree/tooling/trace_replay.c
@@ -48,15 +48,16 @@
iree_status_t status = iree_ok_status();
if (iree_status_is_ok(status)) {
- status = iree_vm_list_create(NULL, 8u, host_allocator, &out_replay->inputs);
+ status = iree_vm_list_create(iree_vm_make_undefined_type_def(), 8u,
+ host_allocator, &out_replay->inputs);
}
if (iree_status_is_ok(status)) {
- status =
- iree_vm_list_create(NULL, 8u, host_allocator, &out_replay->outputs);
+ status = iree_vm_list_create(iree_vm_make_undefined_type_def(), 8u,
+ host_allocator, &out_replay->outputs);
}
if (iree_status_is_ok(status)) {
- status =
- iree_vm_list_create(NULL, 8u, host_allocator, &out_replay->blackboard);
+ status = iree_vm_list_create(iree_vm_make_undefined_type_def(), 8u,
+ host_allocator, &out_replay->blackboard);
}
if (!iree_status_is_ok(status)) {
@@ -689,7 +690,7 @@
document, value_node, IREE_SV("items"), &items_node));
iree_vm_list_t* list = NULL;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(/*element_type=*/NULL,
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
/*initial_capacity=*/8,
replay->host_allocator, &list));
@@ -1252,8 +1253,9 @@
&args_node));
iree_vm_list_t* input_list = NULL;
IREE_RETURN_AND_END_ZONE_IF_ERROR(
- z0, iree_vm_list_create(/*element_type=*/NULL, /*initial_capacity=*/8,
- replay->host_allocator, &input_list));
+ z0, iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ /*initial_capacity=*/8, replay->host_allocator,
+ &input_list));
iree_status_t status = iree_trace_replay_parse_item_sequence(
replay, document, args_node, input_list);
if (iree_status_is_ok(status)) {
@@ -1299,9 +1301,9 @@
&function, &input_list));
iree_vm_list_t* output_list = NULL;
- iree_status_t status =
- iree_vm_list_create(/*element_type=*/NULL, /*initial_capacity=*/8,
- replay->host_allocator, &output_list);
+ iree_status_t status = iree_vm_list_create(
+ iree_vm_make_undefined_type_def(), /*initial_capacity=*/8,
+ replay->host_allocator, &output_list);
if (iree_status_is_ok(status) && hooks && hooks->before) {
status = hooks->before(hooks->user_data, replay, document, event_node,
@@ -1373,8 +1375,8 @@
iree_vm_list_t* list = NULL;
IREE_RETURN_AND_END_ZONE_IF_ERROR(
- z0, iree_vm_list_create(/*element_type=*/NULL, 8u, replay->host_allocator,
- &list));
+ z0, iree_vm_list_create(iree_vm_make_undefined_type_def(), 8u,
+ replay->host_allocator, &list));
iree_status_t status =
iree_trace_replay_parse_item_sequence(replay, document, from_node, list);
diff --git a/runtime/src/iree/tooling/vm_util.c b/runtime/src/iree/tooling/vm_util.c
index c3b6fa3..11d89cf 100644
--- a/runtime/src/iree/tooling/vm_util.c
+++ b/runtime/src/iree/tooling/vm_util.c
@@ -156,9 +156,8 @@
iree_vm_list_t* list = NULL;
IREE_RETURN_AND_END_ZONE_IF_ERROR(
- z0,
- iree_vm_list_create(
- /*element_type=*/NULL, input_strings_count, host_allocator, &list));
+ z0, iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ input_strings_count, host_allocator, &list));
iree_status_t status = iree_tooling_parse_into_variant_list(
device_allocator, input_strings, input_strings_count, host_allocator,
@@ -340,7 +339,7 @@
if (iree_vm_variant_is_empty(variant)) {
return iree_string_builder_append_string(builder, IREE_SV("(null)\n"));
} else if (iree_vm_variant_is_value(variant)) {
- switch (variant.type.value_type) {
+ switch (iree_vm_type_def_as_value(variant.type)) {
IREE_PRINTVARIANT_CASE_I(8, builder, variant)
IREE_PRINTVARIANT_CASE_I(16, builder, variant)
IREE_PRINTVARIANT_CASE_I(32, builder, variant)
@@ -351,7 +350,8 @@
return iree_string_builder_append_string(builder, IREE_SV("?\n"));
}
} else if (iree_vm_variant_is_ref(variant)) {
- iree_string_view_t type_name = iree_vm_ref_type_name(variant.type.ref_type);
+ iree_string_view_t type_name =
+ iree_vm_ref_type_name(iree_vm_type_def_as_ref(variant.type));
IREE_RETURN_IF_ERROR(iree_string_builder_append_string(builder, type_name));
IREE_RETURN_IF_ERROR(
iree_string_builder_append_string(builder, IREE_SV("\n")));
diff --git a/runtime/src/iree/tooling/vm_util_test.cc b/runtime/src/iree/tooling/vm_util_test.cc
index a42cfa7..e0ba45c 100644
--- a/runtime/src/iree/tooling/vm_util_test.cc
+++ b/runtime/src/iree/tooling/vm_util_test.cc
@@ -47,8 +47,8 @@
class VmUtilTest : public ::testing::Test {
protected:
virtual void SetUp() {
- IREE_ASSERT_OK(
- iree_vm_instance_create(iree_allocator_system(), &instance_));
+ IREE_ASSERT_OK(iree_vm_instance_create(
+ IREE_VM_TYPE_CAPACITY_DEFAULT, iree_allocator_system(), &instance_));
IREE_ASSERT_OK(iree_hal_module_register_all_types(instance_));
iree_status_t status = iree_hal_create_device(
iree_hal_available_driver_registry(), IREE_SV("local-sync"),
diff --git a/runtime/src/iree/vm/BUILD.bazel b/runtime/src/iree/vm/BUILD.bazel
index 3969e8d..2145392 100644
--- a/runtime/src/iree/vm/BUILD.bazel
+++ b/runtime/src/iree/vm/BUILD.bazel
@@ -82,6 +82,7 @@
"//runtime/src/iree/base:core_headers",
"//runtime/src/iree/base:tracing",
"//runtime/src/iree/base/internal",
+ "//runtime/src/iree/base/internal:synchronization",
],
)
diff --git a/runtime/src/iree/vm/CMakeLists.txt b/runtime/src/iree/vm/CMakeLists.txt
index 5ce8483..c8ba240 100644
--- a/runtime/src/iree/vm/CMakeLists.txt
+++ b/runtime/src/iree/vm/CMakeLists.txt
@@ -68,6 +68,7 @@
iree::base
iree::base::core_headers
iree::base::internal
+ iree::base::internal::synchronization
iree::base::tracing
PUBLIC
)
diff --git a/runtime/src/iree/vm/buffer.c b/runtime/src/iree/vm/buffer.c
index 3a7e24c..fc4510e 100644
--- a/runtime/src/iree/vm/buffer.c
+++ b/runtime/src/iree/vm/buffer.c
@@ -123,11 +123,11 @@
}
IREE_API_EXPORT void iree_vm_buffer_retain(iree_vm_buffer_t* buffer) {
- iree_vm_ref_object_retain(buffer, &iree_vm_buffer_descriptor);
+ iree_vm_ref_object_retain(buffer, iree_vm_buffer_type());
}
IREE_API_EXPORT void iree_vm_buffer_release(iree_vm_buffer_t* buffer) {
- iree_vm_ref_object_release(buffer, &iree_vm_buffer_descriptor);
+ iree_vm_ref_object_release(buffer, iree_vm_buffer_type());
}
IREE_API_EXPORT iree_status_t iree_vm_buffer_clone(
@@ -315,13 +315,12 @@
}
iree_status_t iree_vm_buffer_register_types(iree_vm_instance_t* instance) {
- if (iree_vm_buffer_descriptor.type != IREE_VM_REF_TYPE_NULL) {
- // Already registered.
- return iree_ok_status();
- }
- iree_vm_buffer_descriptor.destroy = iree_vm_buffer_destroy;
- iree_vm_buffer_descriptor.offsetof_counter =
- offsetof(iree_vm_buffer_t, ref_object.counter);
- iree_vm_buffer_descriptor.type_name = iree_make_cstring_view("vm.buffer");
- return iree_vm_ref_register_type(&iree_vm_buffer_descriptor);
+ static const iree_vm_ref_type_descriptor_t descriptor = {
+ .destroy = iree_vm_buffer_destroy,
+ .type_name = IREE_SVL("vm.buffer"),
+ .offsetof_counter = offsetof(iree_vm_buffer_t, ref_object.counter) /
+ IREE_VM_REF_COUNTER_ALIGNMENT,
+ };
+ return iree_vm_instance_register_type(instance, &descriptor,
+ &iree_vm_buffer_registration);
}
diff --git a/runtime/src/iree/vm/buffer_test.cc b/runtime/src/iree/vm/buffer_test.cc
index ef1c660..d102ef3 100644
--- a/runtime/src/iree/vm/buffer_test.cc
+++ b/runtime/src/iree/vm/buffer_test.cc
@@ -17,7 +17,8 @@
static iree_vm_instance_t* instance = NULL;
struct VMBufferTest : public ::testing::Test {
static void SetUpTestSuite() {
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance));
}
static void TearDownTestSuite() { iree_vm_instance_release(instance); }
};
diff --git a/runtime/src/iree/vm/bytecode/disassembler.c b/runtime/src/iree/vm/bytecode/disassembler.c
index 3bdb3f4..cb9ecdf 100644
--- a/runtime/src/iree/vm/bytecode/disassembler.c
+++ b/runtime/src/iree/vm/bytecode/disassembler.c
@@ -132,10 +132,10 @@
}
static iree_status_t iree_vm_bytecode_disassembler_emit_type_name(
- const iree_vm_type_def_t* type_def, iree_string_builder_t* b) {
+ iree_vm_type_def_t type_def, iree_string_builder_t* b) {
if (iree_vm_type_def_is_value(type_def)) {
const char* type_name;
- switch (type_def->value_type) {
+ switch (iree_vm_type_def_as_value(type_def)) {
case IREE_VM_VALUE_TYPE_I8:
type_name = "i8";
break;
@@ -160,7 +160,8 @@
}
return iree_string_builder_append_cstring(b, type_name);
} else if (iree_vm_type_def_is_ref(type_def)) {
- iree_string_view_t type_name = iree_vm_ref_type_name(type_def->ref_type);
+ iree_string_view_t type_name =
+ iree_vm_ref_type_name(iree_vm_type_def_as_ref(type_def));
return iree_string_builder_append_format(b, "%.*s", (int)type_name.size,
type_name.data);
} else {
@@ -522,7 +523,7 @@
DISASM_OP(CORE, GlobalLoadRef) {
uint32_t global = VM_ParseGlobalAttr("global");
- const iree_vm_type_def_t* type_def = VM_ParseTypeOf("value");
+ const iree_vm_type_def_t type_def = VM_ParseTypeOf("value");
bool result_is_move;
uint16_t result_reg = VM_ParseResultRegRef("value", &result_is_move);
EMIT_REF_REG_NAME(result_reg);
@@ -536,7 +537,7 @@
DISASM_OP(CORE, GlobalStoreRef) {
uint32_t global = VM_ParseGlobalAttr("global");
- const iree_vm_type_def_t* type_def = VM_ParseTypeOf("value");
+ const iree_vm_type_def_t type_def = VM_ParseTypeOf("value");
bool value_is_move;
uint16_t value_reg = VM_ParseOperandRegRef("value", &value_is_move);
IREE_RETURN_IF_ERROR(
@@ -551,7 +552,7 @@
DISASM_OP(CORE, GlobalLoadIndirectRef) {
uint16_t global_reg = VM_ParseOperandRegI32("global");
- const iree_vm_type_def_t* type_def = VM_ParseTypeOf("value");
+ const iree_vm_type_def_t type_def = VM_ParseTypeOf("value");
bool result_is_move;
uint16_t result_reg = VM_ParseResultRegRef("value", &result_is_move);
EMIT_REF_REG_NAME(result_reg);
@@ -569,7 +570,7 @@
DISASM_OP(CORE, GlobalStoreIndirectRef) {
uint16_t global_reg = VM_ParseOperandRegI32("global");
- const iree_vm_type_def_t* type_def = VM_ParseTypeOf("value");
+ const iree_vm_type_def_t type_def = VM_ParseTypeOf("value");
bool value_is_move;
uint16_t value_reg = VM_ParseOperandRegRef("value", &value_is_move);
IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
@@ -1026,7 +1027,7 @@
//===------------------------------------------------------------------===//
DISASM_OP(CORE, ListAlloc) {
- const iree_vm_type_def_t* element_type_def =
+ const iree_vm_type_def_t element_type_def =
VM_ParseTypeOf("element_type");
uint16_t initial_capacity_reg = VM_ParseOperandRegI32("initial_capacity");
bool result_is_move;
@@ -1155,7 +1156,7 @@
bool list_is_move;
uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
uint16_t index_reg = VM_ParseOperandRegI32("index");
- const iree_vm_type_def_t* type_def = VM_ParseTypeOf("result");
+ const iree_vm_type_def_t type_def = VM_ParseTypeOf("result");
bool result_is_move;
uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move);
EMIT_REF_REG_NAME(result_reg);
@@ -1233,7 +1234,7 @@
DISASM_OP(CORE, SelectRef) {
uint16_t condition_reg = VM_ParseOperandRegI32("condition");
- const iree_vm_type_def_t* type_def = VM_ParseTypeOf("true_value");
+ const iree_vm_type_def_t type_def = VM_ParseTypeOf("true_value");
bool true_value_is_move;
uint16_t true_value_reg =
VM_ParseOperandRegRef("true_value", &true_value_is_move);
diff --git a/runtime/src/iree/vm/bytecode/dispatch.c b/runtime/src/iree/vm/bytecode/dispatch.c
index 31473da..e52bc49 100644
--- a/runtime/src/iree/vm/bytecode/dispatch.c
+++ b/runtime/src/iree/vm/bytecode/dispatch.c
@@ -782,23 +782,24 @@
DISPATCH_OP(CORE, GlobalLoadRef, {
uint32_t global = VM_DecGlobalAttr("global");
IREE_ASSERT(global < module_state->global_ref_count);
- const iree_vm_type_def_t* type_def = VM_DecTypeOf("value");
+ const iree_vm_type_def_t type_def = VM_DecTypeOf("value");
bool result_is_move;
iree_vm_ref_t* result = VM_DecResultRegRef("value", &result_is_move);
iree_vm_ref_t* global_ref = &module_state->global_ref_table[global];
IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked(
- result_is_move, global_ref, type_def->ref_type, result));
+ result_is_move, global_ref, iree_vm_type_def_as_ref(type_def),
+ result));
});
DISPATCH_OP(CORE, GlobalStoreRef, {
uint32_t global = VM_DecGlobalAttr("global");
IREE_ASSERT(global < module_state->global_ref_count);
- const iree_vm_type_def_t* type_def = VM_DecTypeOf("value");
+ const iree_vm_type_def_t type_def = VM_DecTypeOf("value");
bool value_is_move;
iree_vm_ref_t* value = VM_DecOperandRegRef("value", &value_is_move);
iree_vm_ref_t* global_ref = &module_state->global_ref_table[global];
IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked(
- value_is_move, value, type_def->ref_type, global_ref));
+ value_is_move, value, iree_vm_type_def_as_ref(type_def), global_ref));
});
DISPATCH_OP(CORE, GlobalLoadIndirectRef, {
@@ -809,12 +810,13 @@
"global ref ordinal out of range: %d (table=%zu)", global,
module_state->global_ref_count);
}
- const iree_vm_type_def_t* type_def = VM_DecTypeOf("value");
+ const iree_vm_type_def_t type_def = VM_DecTypeOf("value");
bool result_is_move;
iree_vm_ref_t* result = VM_DecResultRegRef("value", &result_is_move);
iree_vm_ref_t* global_ref = &module_state->global_ref_table[global];
IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked(
- result_is_move, global_ref, type_def->ref_type, result));
+ result_is_move, global_ref, iree_vm_type_def_as_ref(type_def),
+ result));
});
DISPATCH_OP(CORE, GlobalStoreIndirectRef, {
@@ -825,12 +827,12 @@
"global ref ordinal out of range: %d (table=%zu)", global,
module_state->global_ref_count);
}
- const iree_vm_type_def_t* type_def = VM_DecTypeOf("value");
+ const iree_vm_type_def_t type_def = VM_DecTypeOf("value");
bool value_is_move;
iree_vm_ref_t* value = VM_DecOperandRegRef("value", &value_is_move);
iree_vm_ref_t* global_ref = &module_state->global_ref_table[global];
IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked(
- value_is_move, value, type_def->ref_type, global_ref));
+ value_is_move, value, iree_vm_type_def_as_ref(type_def), global_ref));
});
//===------------------------------------------------------------------===//
@@ -872,7 +874,7 @@
iree_vm_ref_t* result = VM_DecResultRegRef("value", &result_is_move);
IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_retain(
&module_state->rodata_ref_table[rodata_ordinal],
- iree_vm_buffer_type_id(), result));
+ iree_vm_buffer_type(), result));
});
//===------------------------------------------------------------------===//
@@ -888,8 +890,8 @@
IREE_RETURN_IF_ERROR(iree_vm_buffer_create(
IREE_VM_BUFFER_ACCESS_MUTABLE | IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST,
length, alignment, module_state->allocator, &buffer));
- IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_assign(
- buffer, iree_vm_buffer_type_id(), result_ref));
+ IREE_RETURN_IF_ERROR(
+ iree_vm_ref_wrap_assign(buffer, iree_vm_buffer_type(), result_ref));
});
DISPATCH_OP(CORE, BufferClone, {
@@ -909,8 +911,8 @@
IREE_RETURN_IF_ERROR(iree_vm_buffer_clone(
IREE_VM_BUFFER_ACCESS_MUTABLE | IREE_VM_BUFFER_ACCESS_ORIGIN_GUEST,
source, offset, length, alignment, module_state->allocator, &result));
- IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_assign(
- result, iree_vm_buffer_type_id(), result_ref));
+ IREE_RETURN_IF_ERROR(
+ iree_vm_ref_wrap_assign(result, iree_vm_buffer_type(), result_ref));
});
DISPATCH_OP(CORE, BufferLength, {
@@ -1179,7 +1181,7 @@
//===------------------------------------------------------------------===//
DISPATCH_OP(CORE, ListAlloc, {
- const iree_vm_type_def_t* element_type_def = VM_DecTypeOf("element_type");
+ const iree_vm_type_def_t element_type_def = VM_DecTypeOf("element_type");
uint32_t initial_capacity = VM_DecOperandRegI32("initial_capacity");
bool result_is_move;
iree_vm_ref_t* result = VM_DecResultRegRef("result", &result_is_move);
@@ -1187,7 +1189,7 @@
IREE_RETURN_IF_ERROR(iree_vm_list_create(
element_type_def, initial_capacity, module_state->allocator, &list));
IREE_RETURN_IF_ERROR(
- iree_vm_ref_wrap_assign(list, iree_vm_list_type_id(), result));
+ iree_vm_ref_wrap_assign(list, iree_vm_list_type(), result));
});
DISPATCH_OP(CORE, ListReserve, {
@@ -1287,14 +1289,14 @@
return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, "list is null");
}
uint32_t index = VM_DecOperandRegI32("index");
- const iree_vm_type_def_t* type_def = VM_DecTypeOf("result");
+ const iree_vm_type_def_t type_def = VM_DecTypeOf("result");
bool result_is_move;
iree_vm_ref_t* result = VM_DecResultRegRef("result", &result_is_move);
// TODO(benvanik): use result_is_move with a _retain_or_move.
IREE_RETURN_IF_ERROR(iree_vm_list_get_ref_retain(list, index, result));
if (result->type != IREE_VM_REF_TYPE_NULL &&
(iree_vm_type_def_is_value(type_def) ||
- result->type != type_def->ref_type)) {
+ result->type != iree_vm_type_def_as_ref(type_def))) {
// Type mismatch; put null in the register instead.
// TODO(benvanik): return an error here and make a query type method?
iree_vm_ref_release(result);
@@ -1342,7 +1344,7 @@
int32_t condition = VM_DecOperandRegI32("condition");
// TODO(benvanik): remove the type_id and use either LHS/RHS (if both are
// null then output is always null so no need to know the type).
- const iree_vm_type_def_t* type_def = VM_DecTypeOf("true_value");
+ const iree_vm_type_def_t type_def = VM_DecTypeOf("true_value");
bool true_value_is_move;
iree_vm_ref_t* true_value =
VM_DecOperandRegRef("true_value", &true_value_is_move);
@@ -1354,14 +1356,16 @@
if (condition) {
// Select LHS.
IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked(
- true_value_is_move, true_value, type_def->ref_type, result));
+ true_value_is_move, true_value, iree_vm_type_def_as_ref(type_def),
+ result));
if (false_value_is_move && false_value != result) {
iree_vm_ref_release(false_value);
}
} else {
// Select RHS.
IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked(
- false_value_is_move, false_value, type_def->ref_type, result));
+ false_value_is_move, false_value, iree_vm_type_def_as_ref(type_def),
+ result));
if (true_value_is_move && true_value != result) {
iree_vm_ref_release(true_value);
}
@@ -1396,7 +1400,7 @@
DISPATCH_OP(CORE, SwitchRef, {
int32_t index = VM_DecOperandRegI32("index");
- const iree_vm_type_def_t* type_def = VM_DecTypeOf("result");
+ const iree_vm_type_def_t type_def = VM_DecTypeOf("result");
bool default_is_move;
iree_vm_ref_t* default_value =
VM_DecOperandRegRef("default_value", &default_is_move);
@@ -1410,10 +1414,11 @@
iree_vm_ref_t* new_value = ®s_ref[value_reg_list->registers[index] &
IREE_REF_REGISTER_MASK];
IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked(
- is_move, new_value, type_def->ref_type, result));
+ is_move, new_value, iree_vm_type_def_as_ref(type_def), result));
} else {
IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked(
- default_is_move, default_value, type_def->ref_type, result));
+ default_is_move, default_value, iree_vm_type_def_as_ref(type_def),
+ result));
}
});
diff --git a/runtime/src/iree/vm/bytecode/dispatch_async_test.cc b/runtime/src/iree/vm/bytecode/dispatch_async_test.cc
index 9858e49..f2e2ccf 100644
--- a/runtime/src/iree/vm/bytecode/dispatch_async_test.cc
+++ b/runtime/src/iree/vm/bytecode/dispatch_async_test.cc
@@ -30,7 +30,8 @@
IREE_TRACE_SCOPE();
const iree_file_toc_t* file = async_bytecode_modules_c_create();
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance_));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance_));
IREE_CHECK_OK(iree_vm_bytecode_module_create(
instance_,
diff --git a/runtime/src/iree/vm/bytecode/dispatch_test.cc b/runtime/src/iree/vm/bytecode/dispatch_test.cc
index d397611..cdab0e5 100644
--- a/runtime/src/iree/vm/bytecode/dispatch_test.cc
+++ b/runtime/src/iree/vm/bytecode/dispatch_test.cc
@@ -37,7 +37,8 @@
std::vector<TestParams> test_params;
iree_vm_instance_t* instance = NULL;
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance));
const struct iree_file_toc_t* module_file_toc =
all_bytecode_modules_c_create();
@@ -75,7 +76,8 @@
virtual void SetUp() {
const auto& test_params = GetParam();
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance_));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance_));
IREE_CHECK_OK(iree_vm_bytecode_module_create(
instance_,
diff --git a/runtime/src/iree/vm/bytecode/dispatch_util.h b/runtime/src/iree/vm/bytecode/dispatch_util.h
index d460df2..d6d377b 100644
--- a/runtime/src/iree/vm/bytecode/dispatch_util.h
+++ b/runtime/src/iree/vm/bytecode/dispatch_util.h
@@ -87,10 +87,10 @@
} iree_vm_bytecode_frame_storage_t;
// Maps a type ID to a type def with clamping for out of bounds values.
-static inline const iree_vm_type_def_t* iree_vm_map_type(
+static inline iree_vm_type_def_t iree_vm_map_type(
iree_vm_bytecode_module_t* module, int32_t type_id) {
type_id = type_id >= module->type_count ? 0 : type_id;
- return &module->type_table[type_id];
+ return module->type_table[type_id];
}
//===----------------------------------------------------------------------===//
diff --git a/runtime/src/iree/vm/bytecode/module.c b/runtime/src/iree/vm/bytecode/module.c
index 4d88f4c..8060457 100644
--- a/runtime/src/iree/vm/bytecode/module.c
+++ b/runtime/src/iree/vm/bytecode/module.c
@@ -25,39 +25,39 @@
// Resolves a type through either builtin rules or the ref registered types.
static bool iree_vm_bytecode_module_resolve_type(
- iree_vm_TypeDef_table_t type_def, iree_vm_type_def_t* out_type) {
+ iree_vm_instance_t* instance, iree_vm_TypeDef_table_t type_def,
+ iree_vm_type_def_t* out_type) {
memset(out_type, 0, sizeof(*out_type));
flatbuffers_string_t full_name = iree_vm_TypeDef_full_name(type_def);
if (!flatbuffers_string_len(full_name)) {
return false;
} else if (iree_vm_flatbuffer_strcmp(full_name,
iree_make_cstring_view("i8")) == 0) {
- out_type->value_type = IREE_VM_VALUE_TYPE_I8;
+ *out_type = iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I8);
return true;
} else if (iree_vm_flatbuffer_strcmp(full_name,
iree_make_cstring_view("i16")) == 0) {
- out_type->value_type = IREE_VM_VALUE_TYPE_I16;
+ *out_type = iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I16);
return true;
} else if (iree_vm_flatbuffer_strcmp(full_name,
iree_make_cstring_view("i32")) == 0) {
- out_type->value_type = IREE_VM_VALUE_TYPE_I32;
+ *out_type = iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
return true;
} else if (iree_vm_flatbuffer_strcmp(full_name,
iree_make_cstring_view("i64")) == 0) {
- out_type->value_type = IREE_VM_VALUE_TYPE_I64;
+ *out_type = iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I64);
return true;
} else if (iree_vm_flatbuffer_strcmp(full_name,
iree_make_cstring_view("f32")) == 0) {
- out_type->value_type = IREE_VM_VALUE_TYPE_F32;
+ *out_type = iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_F32);
return true;
} else if (iree_vm_flatbuffer_strcmp(full_name,
iree_make_cstring_view("f64")) == 0) {
- out_type->value_type = IREE_VM_VALUE_TYPE_F64;
+ *out_type = iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_F64);
return true;
} else if (iree_vm_flatbuffer_strcmp(
full_name, iree_make_cstring_view("!vm.opaque")) == 0) {
- out_type->value_type = IREE_VM_VALUE_TYPE_NONE;
- out_type->ref_type = IREE_VM_REF_TYPE_NULL;
+ *out_type = iree_vm_make_undefined_type_def();
return true;
} else if (full_name[0] == '!') {
// Note that we drop the ! prefix:
@@ -70,10 +70,9 @@
// all we have registered.
type_name = iree_make_cstring_view("vm.list");
}
- const iree_vm_ref_type_descriptor_t* type_descriptor =
- iree_vm_ref_lookup_registered_type(type_name);
- if (type_descriptor) {
- out_type->ref_type = type_descriptor->type;
+ iree_vm_ref_type_t type = iree_vm_instance_lookup_type(instance, type_name);
+ if (type) {
+ *out_type = iree_vm_make_ref_type_def(type);
return true;
}
}
@@ -84,12 +83,14 @@
// |type_table| can be omitted to just perform verification that all types are
// registered.
static iree_status_t iree_vm_bytecode_module_resolve_types(
- iree_vm_TypeDef_vec_t type_defs, iree_vm_type_def_t* type_table) {
+ iree_vm_instance_t* instance, iree_vm_TypeDef_vec_t type_defs,
+ iree_vm_type_def_t* type_table) {
IREE_TRACE_ZONE_BEGIN(z0);
iree_status_t status = iree_ok_status();
for (size_t i = 0; i < iree_vm_TypeDef_vec_len(type_defs); ++i) {
iree_vm_TypeDef_table_t type_def = iree_vm_TypeDef_vec_at(type_defs, i);
- if (!iree_vm_bytecode_module_resolve_type(type_def, &type_table[i])) {
+ if (!iree_vm_bytecode_module_resolve_type(instance, type_def,
+ &type_table[i])) {
status = iree_make_status(IREE_STATUS_NOT_FOUND,
"no type registered with name '%s'",
iree_vm_TypeDef_full_name(type_def));
@@ -882,8 +883,8 @@
module->def = module_def;
module->type_count = iree_vm_TypeDef_vec_len(type_defs);
- iree_status_t resolve_status =
- iree_vm_bytecode_module_resolve_types(type_defs, module->type_table);
+ iree_status_t resolve_status = iree_vm_bytecode_module_resolve_types(
+ instance, type_defs, module->type_table);
if (!iree_status_is_ok(resolve_status)) {
iree_allocator_free(allocator, module);
IREE_TRACE_ZONE_END(z0);
diff --git a/runtime/src/iree/vm/bytecode/module_benchmark.cc b/runtime/src/iree/vm/bytecode/module_benchmark.cc
index 3069fb5..6d6805e 100644
--- a/runtime/src/iree/vm/bytecode/module_benchmark.cc
+++ b/runtime/src/iree/vm/bytecode/module_benchmark.cc
@@ -76,7 +76,8 @@
std::vector<int32_t> i32_args,
int result_count, int64_t batch_size = 1) {
iree_vm_instance_t* instance = NULL;
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance));
iree_vm_module_t* import_module = NULL;
IREE_CHECK_OK(native_import_module_create(instance, iree_allocator_system(),
@@ -134,7 +135,8 @@
static void BM_ModuleCreate(benchmark::State& state) {
iree_vm_instance_t* instance = NULL;
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance));
while (state.KeepRunning()) {
const auto* module_file_toc =
@@ -159,7 +161,8 @@
static void BM_ModuleCreateState(benchmark::State& state) {
iree_vm_instance_t* instance = NULL;
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance));
const auto* module_file_toc =
iree_vm_bytecode_module_benchmark_module_create();
@@ -189,7 +192,8 @@
static void BM_FullModuleInit(benchmark::State& state) {
iree_vm_instance_t* instance = NULL;
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance));
while (state.KeepRunning()) {
const auto* module_file_toc =
diff --git a/runtime/src/iree/vm/bytecode/module_size_benchmark.cc b/runtime/src/iree/vm/bytecode/module_size_benchmark.cc
index 5aa2fb7..7449173 100644
--- a/runtime/src/iree/vm/bytecode/module_size_benchmark.cc
+++ b/runtime/src/iree/vm/bytecode/module_size_benchmark.cc
@@ -11,7 +11,8 @@
extern "C" int main(int argc, char** argv) {
iree_vm_instance_t* instance = nullptr;
- iree_vm_instance_create(iree_allocator_system(), &instance);
+ iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance);
const auto* module_file_toc =
iree_vm_bytecode_module_size_benchmark_module_create();
diff --git a/runtime/src/iree/vm/bytecode/module_test.cc b/runtime/src/iree/vm/bytecode/module_test.cc
index 270dfc1..f0af597 100644
--- a/runtime/src/iree/vm/bytecode/module_test.cc
+++ b/runtime/src/iree/vm/bytecode/module_test.cc
@@ -110,7 +110,8 @@
class VMBytecodeModuleTest : public ::testing::Test {
protected:
virtual void SetUp() {
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance_));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance_));
const auto* module_file_toc = iree_vm_bytecode_module_test_module_create();
IREE_CHECK_OK(iree_vm_bytecode_module_create(
@@ -136,7 +137,7 @@
const char* function_name, std::vector<iree_vm_value_t> inputs) {
ref<iree_vm_list_t> input_list;
IREE_RETURN_IF_ERROR(
- iree_vm_list_create(/*element_type=*/nullptr, inputs.size(),
+ iree_vm_list_create(iree_vm_make_undefined_type_def(), inputs.size(),
iree_allocator_system(), &input_list));
IREE_RETURN_IF_ERROR(iree_vm_list_resize(input_list.get(), inputs.size()));
for (iree_host_size_t i = 0; i < inputs.size(); ++i) {
@@ -145,8 +146,9 @@
}
ref<iree_vm_list_t> output_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 8, iree_allocator_system(), &output_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 8, iree_allocator_system(),
+ &output_list));
iree_vm_function_t function;
IREE_RETURN_IF_ERROR(iree_vm_module_lookup_function_by_name(
@@ -170,7 +172,7 @@
const char* function_name, std::vector<iree_vm_ref_t> inputs) {
ref<iree_vm_list_t> input_list;
IREE_RETURN_IF_ERROR(
- iree_vm_list_create(/*element_type=*/nullptr, inputs.size(),
+ iree_vm_list_create(iree_vm_make_undefined_type_def(), inputs.size(),
iree_allocator_system(), &input_list));
IREE_RETURN_IF_ERROR(iree_vm_list_resize(input_list.get(), inputs.size()));
for (iree_host_size_t i = 0; i < inputs.size(); ++i) {
@@ -179,8 +181,9 @@
}
ref<iree_vm_list_t> output_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 8, iree_allocator_system(), &output_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 8, iree_allocator_system(),
+ &output_list));
iree_vm_function_t function;
IREE_RETURN_IF_ERROR(iree_vm_module_lookup_function_by_name(
diff --git a/runtime/src/iree/vm/context.c b/runtime/src/iree/vm/context.c
index c287b13..8c0407c 100644
--- a/runtime/src/iree/vm/context.c
+++ b/runtime/src/iree/vm/context.c
@@ -381,11 +381,15 @@
}
}
+IREE_API_EXPORT iree_vm_instance_t* iree_vm_context_instance(
+ const iree_vm_context_t* context) {
+ IREE_ASSERT_ARGUMENT(context);
+ return context->instance;
+}
+
IREE_API_EXPORT iree_vm_context_id_t
iree_vm_context_id(const iree_vm_context_t* context) {
- if (!context) {
- return -1;
- }
+ if (!context) return -1;
return context->context_id;
}
diff --git a/runtime/src/iree/vm/context.h b/runtime/src/iree/vm/context.h
index 1370ac5..b9640ae 100644
--- a/runtime/src/iree/vm/context.h
+++ b/runtime/src/iree/vm/context.h
@@ -75,6 +75,10 @@
// Releases the given |context| from the caller.
IREE_API_EXPORT void iree_vm_context_release(iree_vm_context_t* context);
+// Returns the instance this context was created within.
+IREE_API_EXPORT iree_vm_instance_t* iree_vm_context_instance(
+ const iree_vm_context_t* context);
+
// Returns a process-unique ID for the |context|.
IREE_API_EXPORT iree_vm_context_id_t
iree_vm_context_id(const iree_vm_context_t* context);
diff --git a/runtime/src/iree/vm/instance.c b/runtime/src/iree/vm/instance.c
index 2defba9..ef3e4b0 100644
--- a/runtime/src/iree/vm/instance.c
+++ b/runtime/src/iree/vm/instance.c
@@ -9,6 +9,7 @@
#include <stddef.h>
#include "iree/base/internal/atomics.h"
+#include "iree/base/internal/synchronization.h"
#include "iree/base/tracing.h"
// Defined in their respective files:
@@ -24,23 +25,40 @@
return iree_ok_status();
}
+typedef struct {
+ // Pointer to the externally-defined/unowned descriptor.
+ const iree_vm_ref_type_descriptor_t* descriptor;
+ // Number of times the type has been registered. Only to be used by the
+ // instance and while holding the instance lock.
+ uint32_t registration_count;
+} iree_vm_registered_type_t;
+
struct iree_vm_instance_t {
iree_atomic_ref_count_t ref_count;
iree_allocator_t allocator;
+
+ iree_slim_mutex_t type_mutex;
+ uint16_t type_capacity;
+ uint16_t type_count;
+ iree_vm_registered_type_t types[];
};
IREE_API_EXPORT iree_status_t iree_vm_instance_create(
- iree_allocator_t allocator, iree_vm_instance_t** out_instance) {
+ iree_host_size_t type_capacity, iree_allocator_t allocator,
+ iree_vm_instance_t** out_instance) {
IREE_TRACE_ZONE_BEGIN(z0);
IREE_ASSERT_ARGUMENT(out_instance);
*out_instance = NULL;
iree_vm_instance_t* instance = NULL;
+ iree_host_size_t total_size =
+ sizeof(*instance) + type_capacity * sizeof(instance->types[0]);
IREE_RETURN_AND_END_ZONE_IF_ERROR(
- z0,
- iree_allocator_malloc(allocator, sizeof(*instance), (void**)&instance));
+ z0, iree_allocator_malloc(allocator, total_size, (void**)&instance));
instance->allocator = allocator;
iree_atomic_ref_count_init(&instance->ref_count);
+ iree_slim_mutex_initialize(&instance->type_mutex);
+ instance->type_capacity = type_capacity;
iree_status_t status = iree_vm_register_builtin_types(instance);
@@ -56,6 +74,7 @@
static void iree_vm_instance_destroy(iree_vm_instance_t* instance) {
IREE_TRACE_ZONE_BEGIN(z0);
IREE_ASSERT_ARGUMENT(instance);
+ iree_slim_mutex_deinitialize(&instance->type_mutex);
iree_allocator_free(instance->allocator, instance);
IREE_TRACE_ZONE_END(z0);
}
@@ -77,3 +96,109 @@
IREE_ASSERT_ARGUMENT(instance);
return instance->allocator;
}
+
+IREE_API_EXPORT iree_status_t
+iree_vm_instance_register_type(iree_vm_instance_t* instance,
+ const iree_vm_ref_type_descriptor_t* descriptor,
+ iree_vm_ref_type_t* out_registration) {
+ IREE_ASSERT_ARGUMENT(instance);
+ IREE_ASSERT_ARGUMENT(descriptor);
+ IREE_ASSERT_ARGUMENT(out_registration);
+ *out_registration = 0;
+
+ if ((((uintptr_t)descriptor) & IREE_VM_REF_TYPE_TAG_BIT_MASK) != 0) {
+ return iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
+ "type descriptors must be aligned to %d bytes",
+ (1 << IREE_VM_REF_TYPE_TAG_BITS));
+ }
+
+ if (descriptor->offsetof_counter & ~IREE_VM_REF_TYPE_TAG_BIT_MASK) {
+ return iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
+ "types must have offsets within the first few "
+ "words of their structures");
+ }
+
+ iree_slim_mutex_lock(&instance->type_mutex);
+
+ // Scan to see if there are any existing types registered with this
+ // descriptor.
+ for (iree_host_size_t i = 0; i < instance->type_count; ++i) {
+ iree_vm_registered_type_t* type = &instance->types[i];
+ if (type->descriptor == descriptor) {
+ // Already registered - increment count so that we have a balanced
+ // register/unregister set.
+ ++type->registration_count;
+ iree_slim_mutex_unlock(&instance->type_mutex);
+ *out_registration = (iree_vm_ref_type_t)descriptor |
+ (iree_vm_ref_type_t)descriptor->offsetof_counter;
+ return iree_ok_status();
+ }
+ }
+
+ // Ensure there's capacity.
+ if (instance->type_count + 1 > instance->type_capacity) {
+ iree_slim_mutex_unlock(&instance->type_mutex);
+ return iree_make_status(IREE_STATUS_RESOURCE_EXHAUSTED,
+ "too many user-defined types registered; new type "
+ "%.*s would exceed capacity of %d",
+ (int)descriptor->type_name.size,
+ descriptor->type_name.data,
+ instance->type_capacity);
+ }
+
+ // Append to the list.
+ instance->types[instance->type_count++] = (iree_vm_registered_type_t){
+ .descriptor = descriptor,
+ .registration_count = 1,
+ };
+
+ iree_slim_mutex_unlock(&instance->type_mutex);
+
+ *out_registration = (iree_vm_ref_type_t)descriptor |
+ (iree_vm_ref_type_t)descriptor->offsetof_counter;
+ return iree_ok_status();
+}
+
+IREE_API_EXPORT void iree_vm_instance_unregister_type(
+ iree_vm_instance_t* instance,
+ const iree_vm_ref_type_descriptor_t* descriptor) {
+ iree_slim_mutex_lock(&instance->type_mutex);
+ for (iree_host_size_t i = 0; i < instance->type_count; ++i) {
+ // NOTE: descriptor pointers must be stable so we can just compare that
+ // instead of each field.
+ iree_vm_registered_type_t* type = &instance->types[i];
+ if (type->descriptor == descriptor) {
+ if (--type->registration_count == 0) {
+ // Last registration reference, remove from the list.
+ memmove(&instance->types[i], &instance->types[i + 1],
+ instance->type_count - i - 1);
+ instance->types[--instance->type_count] = (iree_vm_registered_type_t){
+ .descriptor = NULL,
+ .registration_count = 0,
+ };
+ }
+ break;
+ }
+ }
+ iree_slim_mutex_unlock(&instance->type_mutex);
+}
+
+// NOTE: this does a linear scan over the type descriptors even though they are
+// likely in a random order. Type lookup should be done once and reused so this
+// shouldn't really matter.
+IREE_API_EXPORT iree_vm_ref_type_t iree_vm_instance_lookup_type(
+ iree_vm_instance_t* instance, iree_string_view_t full_name) {
+ const iree_vm_ref_type_descriptor_t* descriptor = NULL;
+ iree_slim_mutex_lock(&instance->type_mutex);
+ for (iree_host_size_t i = 0; i < instance->type_count; ++i) {
+ const iree_vm_registered_type_t* type = &instance->types[i];
+ if (iree_string_view_equal(type->descriptor->type_name, full_name)) {
+ descriptor = type->descriptor;
+ break;
+ }
+ }
+ iree_slim_mutex_unlock(&instance->type_mutex);
+ if (!descriptor) return 0;
+ return (iree_vm_ref_type_t)descriptor |
+ (iree_vm_ref_type_t)descriptor->offsetof_counter;
+}
diff --git a/runtime/src/iree/vm/instance.h b/runtime/src/iree/vm/instance.h
index a1f4a1a..fe7ea01 100644
--- a/runtime/src/iree/vm/instance.h
+++ b/runtime/src/iree/vm/instance.h
@@ -14,6 +14,11 @@
extern "C" {
#endif // __cplusplus
+// Default maximum registered types value for iree_vm_instance_create.
+// Users wanting to conserve memory can reduce this to the minimum their
+// instance requires.
+#define IREE_VM_TYPE_CAPACITY_DEFAULT 32
+
// Shared runtime instance responsible for routing iree_vm_context_ events,
// enumerating and creating hardware device interfaces, and managing device
// resource pools.
@@ -30,9 +35,15 @@
// Creates a new instance. This should be shared with all contexts in an
// application to ensure that resources are tracked properly and threads are
// managed correctly.
+//
+// |type_capacity| specifies the maximum number of types that can be
+// simultaneously registered with the instance. Callers can use
+// IREE_VM_TYPE_CAPACITY_DEFAULT if they have a reasonable number of types.
+//
// |out_instance| must be released by the caller.
IREE_API_EXPORT iree_status_t iree_vm_instance_create(
- iree_allocator_t allocator, iree_vm_instance_t** out_instance);
+ iree_host_size_t type_capacity, iree_allocator_t allocator,
+ iree_vm_instance_t** out_instance);
// Retains the given |instance| for the caller.
IREE_API_EXPORT void iree_vm_instance_retain(iree_vm_instance_t* instance);
@@ -44,6 +55,40 @@
IREE_API_EXPORT iree_allocator_t
iree_vm_instance_allocator(iree_vm_instance_t* instance);
+// Registers a user-defined type with the IREE C ref system.
+// The provided destroy function will be used to destroy objects when their
+// reference count goes to 0. NULL can be used to no-op the destruction if the
+// type is not owned by the VM.
+//
+// Descriptors registered multiple times will be deduplicated and counted to
+// ensure a matching number of unregisters are required to fully unregister
+// the type. Descriptors do not need to be unregistered before instance
+// destruction and are only required if the memory defining the descriptor is
+// invalid (shared library unload, etc).
+//
+// Once registered the descriptor must stay valid until all ref types created
+// using it have expired and the type is unregistered from the instance.
+//
+// Upon successful registration |out_registration| will be set to the canonical
+// iree_vm_ref_type_t that should be used when interacting with the type.
+//
+// NOTE: the name is not retained and must be kept live by the caller. Ideally
+// it is stored in static read-only memory in the binary.
+IREE_API_EXPORT iree_status_t
+iree_vm_instance_register_type(iree_vm_instance_t* instance,
+ const iree_vm_ref_type_descriptor_t* descriptor,
+ iree_vm_ref_type_t* out_registration);
+
+// Unregisters a user-defined type with the IREE C ref system.
+// No iree_vm_ref_t instances must be live in the program referencing the type.
+IREE_API_EXPORT void iree_vm_instance_unregister_type(
+ iree_vm_instance_t* instance,
+ const iree_vm_ref_type_descriptor_t* descriptor);
+
+// Returns the registered type descriptor for the given type, if found.
+IREE_API_EXPORT iree_vm_ref_type_t iree_vm_instance_lookup_type(
+ iree_vm_instance_t* instance, iree_string_view_t full_name);
+
#ifdef __cplusplus
} // extern "C"
#endif // __cplusplus
diff --git a/runtime/src/iree/vm/list.c b/runtime/src/iree/vm/list.c
index cc4b703..2d6127b 100644
--- a/runtime/src/iree/vm/list.c
+++ b/runtime/src/iree/vm/list.c
@@ -14,7 +14,7 @@
#include "iree/base/tracing.h"
#include "iree/vm/instance.h"
-static uint8_t iree_vm_value_type_size(iree_vm_value_type_t type) {
+static uint8_t iree_vm_value_type_size(iree_vm_type_def_t type) {
// Size of each iree_vm_value_type_t in bytes. We bitpack these so that we
// can do a simple shift and mask to get the size.
const uint32_t kValueTypeSizes = (0u << 0) | // IREE_VM_VALUE_TYPE_NONE
@@ -25,7 +25,8 @@
(4u << 20) | // IREE_VM_VALUE_TYPE_F32
(8u << 24) | // IREE_VM_VALUE_TYPE_F64
(0u << 28); // unused
- return (kValueTypeSizes >> ((type & 0x7) * 4)) & 0xF;
+ return (kValueTypeSizes >> ((iree_vm_type_def_as_value(type) & 0x7) * 4)) &
+ 0xF;
}
// Defines how the iree_vm_list_t storage is allocated and what elements are
@@ -81,7 +82,7 @@
case IREE_VM_LIST_STORAGE_MODE_VARIANT: {
iree_vm_variant_t* variant_storage = (iree_vm_variant_t*)list->storage;
for (iree_host_size_t i = offset; i < offset + length; ++i) {
- if (iree_vm_type_def_is_ref(&variant_storage[i].type)) {
+ if (iree_vm_type_def_is_ref(variant_storage[i].type)) {
iree_vm_ref_retain_inplace(&variant_storage[i].ref);
}
}
@@ -110,7 +111,7 @@
case IREE_VM_LIST_STORAGE_MODE_VARIANT: {
iree_vm_variant_t* variant_storage = (iree_vm_variant_t*)list->storage;
for (iree_host_size_t i = offset; i < offset + length; ++i) {
- if (iree_vm_type_def_is_ref(&variant_storage[i].type)) {
+ if (iree_vm_type_def_is_ref(variant_storage[i].type)) {
iree_vm_ref_release(&variant_storage[i].ref);
memset(&variant_storage[i].type, 0, sizeof(variant_storage[i].type));
} else {
@@ -126,9 +127,9 @@
const iree_vm_type_def_t* element_type, iree_host_size_t capacity) {
iree_host_size_t element_size = sizeof(iree_vm_variant_t);
if (element_type) {
- if (iree_vm_type_def_is_value(element_type)) {
- element_size = iree_vm_value_type_size(element_type->value_type);
- } else if (iree_vm_type_def_is_ref(element_type)) {
+ if (iree_vm_type_def_is_value(*element_type)) {
+ element_size = iree_vm_value_type_size(*element_type);
+ } else if (iree_vm_type_def_is_ref(*element_type)) {
element_size = sizeof(iree_vm_ref_t);
} else {
element_size = sizeof(iree_vm_variant_t);
@@ -146,10 +147,10 @@
iree_vm_list_storage_mode_t storage_mode = IREE_VM_LIST_STORAGE_MODE_VARIANT;
iree_host_size_t element_size = sizeof(iree_vm_variant_t);
if (element_type) {
- if (iree_vm_type_def_is_value(element_type)) {
+ if (iree_vm_type_def_is_value(*element_type)) {
storage_mode = IREE_VM_LIST_STORAGE_MODE_VALUE;
- element_size = iree_vm_value_type_size(element_type->value_type);
- } else if (iree_vm_type_def_is_ref(element_type)) {
+ element_size = iree_vm_value_type_size(*element_type);
+ } else if (iree_vm_type_def_is_ref(*element_type)) {
storage_mode = IREE_VM_LIST_STORAGE_MODE_REF;
element_size = sizeof(iree_vm_ref_t);
} else {
@@ -196,7 +197,7 @@
}
IREE_API_EXPORT iree_status_t iree_vm_list_create(
- const iree_vm_type_def_t* element_type, iree_host_size_t initial_capacity,
+ const iree_vm_type_def_t element_type, iree_host_size_t initial_capacity,
iree_allocator_t allocator, iree_vm_list_t** out_list) {
IREE_ASSERT_ARGUMENT(out_list);
IREE_TRACE_ZONE_BEGIN(z0);
@@ -207,14 +208,12 @@
memset(list, 0, sizeof(*list));
iree_atomic_ref_count_init(&list->ref_object.counter);
list->allocator = allocator;
- if (element_type) {
- list->element_type = *element_type;
- }
+ list->element_type = element_type;
- if (iree_vm_type_def_is_value(&list->element_type) && element_type) {
+ if (iree_vm_type_def_is_value(list->element_type)) {
list->storage_mode = IREE_VM_LIST_STORAGE_MODE_VALUE;
- list->element_size = iree_vm_value_type_size(element_type->value_type);
- } else if (iree_vm_type_def_is_ref(&list->element_type)) {
+ list->element_size = iree_vm_value_type_size(list->element_type);
+ } else if (iree_vm_type_def_is_ref(list->element_type)) {
list->storage_mode = IREE_VM_LIST_STORAGE_MODE_REF;
list->element_size = sizeof(iree_vm_ref_t);
} else {
@@ -252,7 +251,7 @@
iree_vm_type_def_t element_type = iree_vm_list_element_type(source);
iree_vm_list_t* target = NULL;
IREE_RETURN_AND_END_ZONE_IF_ERROR(
- z0, iree_vm_list_create(&element_type, count, host_allocator, &target));
+ z0, iree_vm_list_create(element_type, count, host_allocator, &target));
iree_status_t status = iree_vm_list_resize(target, count);
if (iree_status_is_ok(status)) {
// Copy storage directly. Note that we need to retain any refs contained.
@@ -270,11 +269,11 @@
}
IREE_API_EXPORT void iree_vm_list_retain(iree_vm_list_t* list) {
- iree_vm_ref_object_retain(list, &iree_vm_list_descriptor);
+ iree_vm_ref_object_retain(list, iree_vm_list_type());
}
IREE_API_EXPORT void iree_vm_list_release(iree_vm_list_t* list) {
- iree_vm_ref_object_release(list, &iree_vm_list_descriptor);
+ iree_vm_ref_object_release(list, iree_vm_list_type());
}
IREE_API_EXPORT iree_vm_type_def_t
@@ -362,12 +361,6 @@
iree_memswap(&list_a->storage, &list_b->storage, sizeof(list_a->storage));
}
-// Returns true if |src_type| can be converted into |dst_type|.
-static bool iree_vm_type_def_is_compatible(iree_vm_type_def_t src_type,
- iree_vm_type_def_t dst_type) {
- return memcmp(&src_type, &dst_type, sizeof(dst_type)) == 0;
-}
-
// Copies from a |src_list| of any type (value, ref, variant) into a |dst_list|
// in variant storage mode. This cannot fail as variant lists can store any
// type.
@@ -383,7 +376,7 @@
uintptr_t src_storage =
(uintptr_t)src_list->storage + src_i * src_list->element_size;
for (iree_host_size_t i = 0; i < count; ++i) {
- if (iree_vm_type_def_is_ref(&dst_storage[i].type)) {
+ if (iree_vm_type_def_is_ref(dst_storage[i].type)) {
iree_vm_ref_release(&dst_storage[i].ref);
}
dst_storage[i].type = src_list->element_type;
@@ -400,10 +393,10 @@
// same.
iree_vm_ref_t* ref = &src_storage[i];
iree_vm_ref_retain_inplace(ref);
- if (iree_vm_type_def_is_ref(&dst_storage[i].type)) {
+ if (iree_vm_type_def_is_ref(dst_storage[i].type)) {
iree_vm_ref_release(&dst_storage[i].ref);
}
- dst_storage->type = iree_vm_type_def_make_ref_type(ref->type);
+ dst_storage->type = iree_vm_make_ref_type_def(ref->type);
dst_storage->ref = *ref;
}
break;
@@ -414,10 +407,10 @@
for (iree_host_size_t i = 0; i < count; ++i) {
// NOTE: we retain first in case the lists alias and the ref is the
// same.
- if (iree_vm_type_def_is_ref(&src_storage[i].type)) {
+ if (iree_vm_type_def_is_ref(src_storage[i].type)) {
iree_vm_ref_retain_inplace(&src_storage[i].ref);
}
- if (iree_vm_type_def_is_ref(&dst_storage[i].type)) {
+ if (iree_vm_type_def_is_ref(dst_storage[i].type)) {
iree_vm_ref_release(&dst_storage[i].ref);
}
memcpy(&dst_storage[i], &src_storage[i], sizeof(dst_storage[i]));
@@ -441,8 +434,8 @@
case IREE_VM_LIST_STORAGE_MODE_VALUE:
case IREE_VM_LIST_STORAGE_MODE_REF:
for (iree_host_size_t i = 0; i < count; ++i) {
- if (!iree_vm_type_def_is_compatible(src_storage[i].type,
- dst_list->element_type)) {
+ if (!iree_vm_type_def_equal(src_storage[i].type,
+ dst_list->element_type)) {
return iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
"destination list element type does not "
"match the source element %" PRIhsz,
@@ -532,8 +525,7 @@
// If neither source or destination are variant lists we need to match the
// types exactly.
if (src_list->storage_mode != dst_list->storage_mode ||
- memcmp(&src_list->element_type, &dst_list->element_type,
- sizeof(src_list->element_type)) != 0) {
+ !iree_vm_type_def_equal(src_list->element_type, dst_list->element_type)) {
return iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
"src/dst element type mismatch");
}
@@ -638,7 +630,7 @@
memset(out_value, 0, sizeof(*out_value));
switch (list->storage_mode) {
case IREE_VM_LIST_STORAGE_MODE_VALUE: {
- out_value->type = list->element_type.value_type;
+ out_value->type = iree_vm_type_def_as_value(list->element_type);
// TODO(benvanik): #ifdef on LITTLE/BIG_ENDIAN and just memcpy.
switch (list->element_size) {
case 1:
@@ -658,11 +650,11 @@
}
case IREE_VM_LIST_STORAGE_MODE_VARIANT: {
iree_vm_variant_t* variant = (iree_vm_variant_t*)element_ptr;
- if (!iree_vm_type_def_is_value(&variant->type)) {
+ if (!iree_vm_type_def_is_value(variant->type)) {
return iree_make_status(IREE_STATUS_FAILED_PRECONDITION,
"variant at index %zu is not a value type", i);
}
- out_value->type = variant->type.value_type;
+ out_value->type = iree_vm_type_def_as_value(variant->type);
memcpy(out_value->value_storage, variant->value_storage,
sizeof(out_value->value_storage));
break;
@@ -685,7 +677,7 @@
value.i64 = 0;
switch (list->storage_mode) {
case IREE_VM_LIST_STORAGE_MODE_VALUE: {
- value.type = list->element_type.value_type;
+ value.type = iree_vm_type_def_as_value(list->element_type);
// TODO(benvanik): #ifdef on LITTLE/BIG_ENDIAN and just memcpy.
switch (list->element_size) {
case 1:
@@ -705,11 +697,11 @@
}
case IREE_VM_LIST_STORAGE_MODE_VARIANT: {
iree_vm_variant_t* variant = (iree_vm_variant_t*)element_ptr;
- if (!iree_vm_type_def_is_value(&variant->type)) {
+ if (!iree_vm_variant_is_value(*variant)) {
return iree_make_status(IREE_STATUS_FAILED_PRECONDITION,
"variant at index %zu is not a value type", i);
}
- value.type = variant->type.value_type;
+ value.type = iree_vm_type_def_as_value(variant->type);
memcpy(value.value_storage, variant->value_storage,
sizeof(value.value_storage));
break;
@@ -731,7 +723,7 @@
iree_vm_value_type_t target_type;
switch (list->storage_mode) {
case IREE_VM_LIST_STORAGE_MODE_VALUE: {
- target_type = list->element_type.value_type;
+ target_type = iree_vm_type_def_as_value(list->element_type);
break;
}
case IREE_VM_LIST_STORAGE_MODE_VARIANT: {
@@ -766,11 +758,10 @@
}
case IREE_VM_LIST_STORAGE_MODE_VARIANT: {
iree_vm_variant_t* variant = (iree_vm_variant_t*)element_ptr;
- if (variant->type.ref_type) {
+ if (iree_vm_variant_is_ref(*variant)) {
iree_vm_ref_release(&variant->ref);
}
- variant->type.value_type = target_type;
- variant->type.ref_type = IREE_VM_REF_TYPE_NULL;
+ variant->type = iree_vm_make_value_type_def(target_type);
memcpy(variant->value_storage, converted_value.value_storage,
sizeof(variant->value_storage));
break;
@@ -789,15 +780,15 @@
return iree_vm_list_set_value(list, i, value);
}
-IREE_API_EXPORT void* iree_vm_list_get_ref_deref(
- const iree_vm_list_t* list, iree_host_size_t i,
- const iree_vm_ref_type_descriptor_t* type_descriptor) {
+IREE_API_EXPORT void* iree_vm_list_get_ref_deref(const iree_vm_list_t* list,
+ iree_host_size_t i,
+ iree_vm_ref_type_t type) {
iree_vm_ref_t value = {0};
iree_status_t status = iree_vm_list_get_ref_assign(list, i, &value);
if (!iree_status_is_ok(iree_status_consume_code(status))) {
return NULL;
}
- status = iree_vm_ref_check(value, type_descriptor->type);
+ status = iree_vm_ref_check(value, type);
if (!iree_status_is_ok(iree_status_consume_code(status))) {
return NULL;
}
@@ -825,7 +816,7 @@
case IREE_VM_LIST_STORAGE_MODE_VARIANT: {
iree_vm_variant_t* variant = (iree_vm_variant_t*)element_ptr;
if (!iree_vm_variant_is_empty(*variant) &&
- !iree_vm_type_def_is_ref(&variant->type)) {
+ !iree_vm_type_def_is_ref(variant->type)) {
return iree_make_status(IREE_STATUS_FAILED_PRECONDITION);
}
is_retain ? iree_vm_ref_retain(&variant->ref, out_value)
@@ -863,16 +854,16 @@
case IREE_VM_LIST_STORAGE_MODE_REF: {
iree_vm_ref_t* element_ref = (iree_vm_ref_t*)element_ptr;
IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked(
- is_move, value, list->element_type.ref_type, element_ref));
+ is_move, value, iree_vm_type_def_as_ref(list->element_type),
+ element_ref));
break;
}
case IREE_VM_LIST_STORAGE_MODE_VARIANT: {
iree_vm_variant_t* variant = (iree_vm_variant_t*)element_ptr;
- if (variant->type.value_type) {
+ if (iree_vm_variant_is_value(*variant)) {
memset(&variant->ref, 0, sizeof(variant->ref));
}
- variant->type.value_type = IREE_VM_VALUE_TYPE_NONE;
- variant->type.ref_type = value->type;
+ variant->type = iree_vm_make_ref_type_def(value->type);
iree_vm_ref_retain_or_move(is_move, value, &variant->ref);
break;
}
@@ -967,15 +958,14 @@
}
case IREE_VM_LIST_STORAGE_MODE_REF: {
iree_vm_ref_t* element_ref = (iree_vm_ref_t*)element_ptr;
- out_variant->type.ref_type = element_ref->type;
- out_variant->type.value_type = IREE_VM_VALUE_TYPE_NONE;
+ out_variant->type = iree_vm_make_ref_type_def(element_ref->type);
iree_vm_list_ref_op(ref_mode, element_ref, &out_variant->ref);
break;
}
case IREE_VM_LIST_STORAGE_MODE_VARIANT: {
iree_vm_variant_t* variant = (iree_vm_variant_t*)element_ptr;
out_variant->type = variant->type;
- if (iree_vm_type_def_is_ref(&variant->type)) {
+ if (iree_vm_type_def_is_ref(variant->type)) {
iree_vm_list_ref_op(ref_mode, &variant->ref, &out_variant->ref);
} else {
memcpy(out_variant->value_storage, variant->value_storage,
@@ -1012,21 +1002,14 @@
static iree_status_t iree_vm_list_set_variant(iree_vm_list_t* list,
iree_host_size_t i, bool is_move,
iree_vm_variant_t* variant) {
- if (iree_vm_type_def_is_variant(&variant->type)) {
+ if (iree_vm_variant_is_value(*variant)) {
iree_vm_value_t value = iree_vm_variant_value(*variant);
return iree_vm_list_set_value(list, i, &value);
- } else if (iree_vm_type_def_is_value(&variant->type)) {
- iree_vm_value_t value = {
- .type = variant->type.value_type,
- };
- memcpy(value.value_storage, variant->value_storage,
- sizeof(value.value_storage));
- return iree_vm_list_set_value(list, i, &value);
- } else if (iree_vm_type_def_is_ref(&variant->type)) {
+ } else if (iree_vm_variant_is_ref(*variant)) {
iree_status_t status =
iree_vm_list_set_ref(list, i, is_move, &variant->ref);
if (iree_status_is_ok(status) && is_move) {
- variant->type.ref_type = IREE_VM_REF_TYPE_NULL;
+ variant->type = iree_vm_make_undefined_type_def();
}
return status;
} else {
@@ -1066,14 +1049,12 @@
}
iree_status_t iree_vm_list_register_types(iree_vm_instance_t* instance) {
- if (iree_vm_list_descriptor.type != IREE_VM_REF_TYPE_NULL) {
- // Already registered.
- return iree_ok_status();
- }
-
- iree_vm_list_descriptor.destroy = iree_vm_list_destroy;
- iree_vm_list_descriptor.offsetof_counter =
- offsetof(iree_vm_list_t, ref_object.counter);
- iree_vm_list_descriptor.type_name = iree_make_cstring_view("vm.list");
- return iree_vm_ref_register_type(&iree_vm_list_descriptor);
+ static const iree_vm_ref_type_descriptor_t descriptor = {
+ .destroy = iree_vm_list_destroy,
+ .type_name = IREE_SVL("vm.list"),
+ .offsetof_counter = offsetof(iree_vm_list_t, ref_object.counter) /
+ IREE_VM_REF_COUNTER_ALIGNMENT,
+ };
+ return iree_vm_instance_register_type(instance, &descriptor,
+ &iree_vm_list_registration);
}
diff --git a/runtime/src/iree/vm/list.h b/runtime/src/iree/vm/list.h
index 3b9b176..34d5369 100644
--- a/runtime/src/iree/vm/list.h
+++ b/runtime/src/iree/vm/list.h
@@ -61,10 +61,10 @@
// and ensure that all elements set match the type or IREE_VM_REF_TYPE_ANY to
// indicate that any ref type is allowed.
//
-// |element_type| can be set to iree_vm_type_def_make_variant_type (or null) to
-// indicate that the list stores variants (each element can differ in type).
+// |element_type| can be set to iree_vm_make_undefined_type_def to indicate that
+// the list stores variants (each element can differ in type).
IREE_API_EXPORT iree_status_t iree_vm_list_create(
- const iree_vm_type_def_t* element_type, iree_host_size_t initial_capacity,
+ const iree_vm_type_def_t element_type, iree_host_size_t initial_capacity,
iree_allocator_t allocator, iree_vm_list_t** out_list);
// Shallowly clones |source| into |out_target|.
@@ -157,11 +157,11 @@
IREE_API_EXPORT iree_status_t
iree_vm_list_push_value(iree_vm_list_t* list, const iree_vm_value_t* value);
-// Returns a dereferenced pointer to the given type if the element at the given
-// index matches the type. Returns NULL on error.
-IREE_API_EXPORT void* iree_vm_list_get_ref_deref(
- const iree_vm_list_t* list, iree_host_size_t i,
- const iree_vm_ref_type_descriptor_t* type_descriptor);
+// Returns a dereferenced pointer to the given type if the element at the
+// given index |i| matches the |type|. Returns NULL on error.
+IREE_API_EXPORT void* iree_vm_list_get_ref_deref(const iree_vm_list_t* list,
+ iree_host_size_t i,
+ iree_vm_ref_type_t type);
// Returns the ref value of the element at the given index.
// The ref will not be retained and must be retained by the caller to extend
diff --git a/runtime/src/iree/vm/list_test.cc b/runtime/src/iree/vm/list_test.cc
index a18c4f5..6a9a3a0 100644
--- a/runtime/src/iree/vm/list_test.cc
+++ b/runtime/src/iree/vm/list_test.cc
@@ -102,11 +102,9 @@
for (iree_host_size_t i = 0; i < result.size(); ++i) {
iree_vm_variant_t variant = iree_vm_variant_empty();
IREE_CHECK_OK(iree_vm_list_get_variant_assign(list, i, &variant));
- if (iree_vm_type_def_is_value(&variant.type)) {
- result[i].type = variant.type.value_type;
- memcpy(result[i].value_storage, variant.value_storage,
- sizeof(result[i].value_storage));
- } else if (iree_vm_type_def_is_ref(&variant.type)) {
+ if (iree_vm_variant_is_value(variant)) {
+ result[i] = iree_vm_variant_value(variant);
+ } else if (iree_vm_variant_is_ref(variant)) {
if (test_a_isa(variant.ref)) {
result[i] = iree_vm_value_make_f32(test_a_deref(variant.ref)->data());
} else if (test_b_isa(variant.ref)) {
@@ -130,19 +128,19 @@
using testing::Eq;
template <typename T>
-static void RegisterRefType(iree_vm_ref_type_descriptor_t* descriptor,
- const char* type_name) {
- if (descriptor->type == IREE_VM_REF_TYPE_NULL) {
- descriptor->type_name = iree_make_cstring_view(type_name);
- descriptor->offsetof_counter = T::offsetof_counter();
- descriptor->destroy = T::DirectDestroy;
- IREE_CHECK_OK(iree_vm_ref_register_type(descriptor));
- }
+static void RegisterRefType(iree_vm_instance_t* instance, const char* type_name,
+ iree_vm_ref_type_t* out_registration) {
+ static iree_vm_ref_type_descriptor_t storage = {0};
+ storage.type_name = iree_make_cstring_view(type_name);
+ storage.offsetof_counter = T::offsetof_counter();
+ storage.destroy = T::DirectDestroy;
+ IREE_CHECK_OK(
+ iree_vm_instance_register_type(instance, &storage, out_registration));
}
static void RegisterRefTypes(iree_vm_instance_t* instance) {
- RegisterRefType<A>(&test_a_descriptor, "AType");
- RegisterRefType<B>(&test_b_descriptor, "BType");
+ RegisterRefType<A>(instance, "AType", &test_a_registration);
+ RegisterRefType<B>(instance, "BType", &test_b_registration);
}
template <typename T, typename V>
@@ -151,14 +149,15 @@
auto* obj = new T();
obj->set_data(value);
IREE_CHECK_OK(iree_vm_ref_wrap_assign(
- obj, iree::vm::ref_type_descriptor<T>::get()->type, &ref));
+ obj, iree::vm::ref_type_descriptor<T>::type(), &ref));
return ref;
}
static iree_vm_instance_t* instance = nullptr;
struct VMListTest : public ::testing::Test {
static void SetUpTestSuite() {
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance));
RegisterRefTypes(instance);
}
static void TearDownTestSuite() { iree_vm_instance_release(instance); }
@@ -168,16 +167,15 @@
// Stores only i32 element types, equivalent to `!vm.list<i32>`.
TEST_F(VMListTest, UsageI32) {
iree_vm_type_def_t element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
iree_host_size_t initial_capacity = 123;
iree_vm_list_t* list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &list));
iree_vm_type_def_t queried_element_type = iree_vm_list_element_type(list);
- EXPECT_TRUE(iree_vm_type_def_is_value(&queried_element_type));
- EXPECT_EQ(0,
- memcmp(&element_type, &queried_element_type, sizeof(element_type)));
+ EXPECT_TRUE(iree_vm_type_def_is_value(queried_element_type));
+ EXPECT_TRUE(iree_vm_type_def_equal(element_type, queried_element_type));
EXPECT_LE(initial_capacity, iree_vm_list_capacity(list));
EXPECT_EQ(0, iree_vm_list_size(list));
@@ -203,17 +201,15 @@
// Tests simple ref object list usage, mainly just for demonstration.
// Stores ref object type A elements only, equivalent to `!vm.list<!vm.ref<A>>`.
TEST_F(VMListTest, UsageRef) {
- iree_vm_type_def_t element_type =
- iree_vm_type_def_make_ref_type(test_a_type_id());
+ iree_vm_type_def_t element_type = iree_vm_make_ref_type_def(test_a_type());
iree_host_size_t initial_capacity = 123;
iree_vm_list_t* list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &list));
iree_vm_type_def_t queried_element_type = iree_vm_list_element_type(list);
- EXPECT_TRUE(iree_vm_type_def_is_ref(&queried_element_type));
- EXPECT_EQ(0,
- memcmp(&element_type, &queried_element_type, sizeof(element_type)));
+ EXPECT_TRUE(iree_vm_type_def_is_ref(queried_element_type));
+ EXPECT_TRUE(iree_vm_type_def_equal(element_type, queried_element_type));
EXPECT_LE(initial_capacity, iree_vm_list_capacity(list));
EXPECT_EQ(0, iree_vm_list_size(list));
@@ -240,14 +236,14 @@
// Tests simple variant list usage, mainly just for demonstration.
// Stores any heterogeneous element type, equivalent to `!vm.list<?>`.
TEST_F(VMListTest, UsageVariant) {
- iree_vm_type_def_t element_type = iree_vm_type_def_make_variant_type();
+ iree_vm_type_def_t element_type = iree_vm_make_undefined_type_def();
iree_host_size_t initial_capacity = 123;
iree_vm_list_t* list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &list));
iree_vm_type_def_t queried_element_type = iree_vm_list_element_type(list);
- EXPECT_TRUE(iree_vm_type_def_is_variant(&queried_element_type));
+ EXPECT_TRUE(iree_vm_type_def_is_variant(queried_element_type));
EXPECT_LE(initial_capacity, iree_vm_list_capacity(list));
EXPECT_EQ(0, iree_vm_list_size(list));
@@ -286,10 +282,10 @@
TEST_F(VMListTest, CloneValuesEmpty) {
// Create source list.
iree_vm_type_def_t element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
iree_host_size_t initial_capacity = 123;
iree_vm_list_t* source_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &source_list));
// Clone list.
@@ -300,9 +296,8 @@
// Verify the target list matches source parameters.
iree_vm_type_def_t queried_element_type =
iree_vm_list_element_type(target_list);
- EXPECT_TRUE(iree_vm_type_def_is_value(&queried_element_type));
- EXPECT_EQ(0,
- memcmp(&element_type, &queried_element_type, sizeof(element_type)));
+ EXPECT_TRUE(iree_vm_type_def_is_value(queried_element_type));
+ EXPECT_TRUE(iree_vm_type_def_equal(element_type, queried_element_type));
EXPECT_LE(iree_vm_list_capacity(target_list),
iree_vm_list_capacity(source_list));
EXPECT_EQ(iree_vm_list_size(target_list), iree_vm_list_size(source_list));
@@ -313,10 +308,10 @@
TEST_F(VMListTest, CloneValues) {
// Create source list.
iree_vm_type_def_t element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
iree_host_size_t initial_capacity = 123;
iree_vm_list_t* source_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &source_list));
IREE_ASSERT_OK(iree_vm_list_resize(source_list, 5));
EXPECT_EQ(5, iree_vm_list_size(source_list));
@@ -346,10 +341,9 @@
// Tests cloning lists of ref types.
TEST_F(VMListTest, CloneRefsEmpty) {
- iree_vm_type_def_t element_type =
- iree_vm_type_def_make_ref_type(test_a_type_id());
+ iree_vm_type_def_t element_type = iree_vm_make_ref_type_def(test_a_type());
iree_vm_list_t* source_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, 8, iree_allocator_system(),
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, 8, iree_allocator_system(),
&source_list));
// Clone list.
@@ -360,9 +354,8 @@
// Verify the target list matches source parameters.
iree_vm_type_def_t queried_element_type =
iree_vm_list_element_type(target_list);
- EXPECT_TRUE(iree_vm_type_def_is_ref(&queried_element_type));
- EXPECT_EQ(0,
- memcmp(&element_type, &queried_element_type, sizeof(element_type)));
+ EXPECT_TRUE(iree_vm_type_def_is_ref(queried_element_type));
+ EXPECT_TRUE(iree_vm_type_def_equal(element_type, queried_element_type));
EXPECT_LE(iree_vm_list_capacity(target_list),
iree_vm_list_capacity(source_list));
EXPECT_EQ(iree_vm_list_size(target_list), iree_vm_list_size(source_list));
@@ -371,10 +364,9 @@
iree_vm_list_release(target_list);
}
TEST_F(VMListTest, CloneRefs) {
- iree_vm_type_def_t element_type =
- iree_vm_type_def_make_ref_type(test_a_type_id());
+ iree_vm_type_def_t element_type = iree_vm_make_ref_type_def(test_a_type());
iree_vm_list_t* source_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, 8, iree_allocator_system(),
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, 8, iree_allocator_system(),
&source_list));
IREE_ASSERT_OK(iree_vm_list_resize(source_list, 5));
EXPECT_EQ(5, iree_vm_list_size(source_list));
@@ -409,9 +401,9 @@
// Tests cloning lists of variant types.
TEST_F(VMListTest, CloneVariantsEmpty) {
- iree_vm_type_def_t element_type = iree_vm_type_def_make_variant_type();
+ iree_vm_type_def_t element_type = iree_vm_make_undefined_type_def();
iree_vm_list_t* source_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, 10, iree_allocator_system(),
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, 10, iree_allocator_system(),
&source_list));
// Clone list.
@@ -422,9 +414,8 @@
// Verify the target list matches source parameters.
iree_vm_type_def_t queried_element_type =
iree_vm_list_element_type(target_list);
- EXPECT_TRUE(iree_vm_type_def_is_variant(&queried_element_type));
- EXPECT_EQ(0,
- memcmp(&element_type, &queried_element_type, sizeof(element_type)));
+ EXPECT_TRUE(iree_vm_type_def_is_variant(queried_element_type));
+ EXPECT_TRUE(iree_vm_type_def_equal(element_type, queried_element_type));
EXPECT_LE(iree_vm_list_capacity(target_list),
iree_vm_list_capacity(source_list));
EXPECT_EQ(iree_vm_list_size(target_list), iree_vm_list_size(source_list));
@@ -433,9 +424,9 @@
iree_vm_list_release(target_list);
}
TEST_F(VMListTest, CloneVariants) {
- iree_vm_type_def_t element_type = iree_vm_type_def_make_variant_type();
+ iree_vm_type_def_t element_type = iree_vm_make_undefined_type_def();
iree_vm_list_t* source_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, 10, iree_allocator_system(),
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, 10, iree_allocator_system(),
&source_list));
IREE_ASSERT_OK(iree_vm_list_resize(source_list, 10));
EXPECT_EQ(10, iree_vm_list_size(source_list));
@@ -481,10 +472,10 @@
// Tests capacity reservation.
TEST_F(VMListTest, Reserve) {
// Allocate with 0 initial capacity (which may get rounded up).
- iree_vm_type_def_t element_type = iree_vm_type_def_make_variant_type();
+ iree_vm_type_def_t element_type = iree_vm_make_undefined_type_def();
iree_host_size_t initial_capacity = 0;
iree_vm_list_t* list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &list));
EXPECT_LE(initial_capacity, iree_vm_list_capacity(list));
EXPECT_EQ(0, iree_vm_list_size(list));
@@ -509,10 +500,10 @@
// Tests the behavior of resize for truncation and extension on primitives.
TEST_F(VMListTest, ResizeI32) {
iree_vm_type_def_t element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
iree_host_size_t initial_capacity = 4;
iree_vm_list_t* list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &list));
EXPECT_LE(initial_capacity, iree_vm_list_capacity(list));
EXPECT_EQ(0, iree_vm_list_size(list));
@@ -559,11 +550,10 @@
// Tests the behavior of resize for truncation and extension on refs.
TEST_F(VMListTest, ResizeRef) {
- iree_vm_type_def_t element_type =
- iree_vm_type_def_make_ref_type(test_a_type_id());
+ iree_vm_type_def_t element_type = iree_vm_make_ref_type_def(test_a_type());
iree_host_size_t initial_capacity = 4;
iree_vm_list_t* list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &list));
EXPECT_LE(initial_capacity, iree_vm_list_capacity(list));
EXPECT_EQ(0, iree_vm_list_size(list));
@@ -610,10 +600,10 @@
// Tests the behavior of resize for truncation and extension on variants.
TEST_F(VMListTest, ResizeVariant) {
- iree_vm_type_def_t element_type = iree_vm_type_def_make_variant_type();
+ iree_vm_type_def_t element_type = iree_vm_make_undefined_type_def();
iree_host_size_t initial_capacity = 4;
iree_vm_list_t* list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &list));
EXPECT_LE(initial_capacity, iree_vm_list_capacity(list));
EXPECT_EQ(0, iree_vm_list_size(list));
@@ -664,10 +654,9 @@
// Tests that swapping the storage of a list with itself is a no-op.
TEST_F(VMListTest, SwapStorageSelf) {
- iree_vm_type_def_t element_type =
- iree_vm_type_def_make_ref_type(test_a_type_id());
+ iree_vm_type_def_t element_type = iree_vm_make_ref_type_def(test_a_type());
iree_vm_list_t* list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, /*initial_capacity=*/8,
iree_allocator_system(), &list));
for (iree_host_size_t i = 0; i < 5; ++i) {
iree_vm_ref_t ref_a = MakeRef<A>((float)i);
@@ -691,10 +680,9 @@
// Tests swapping the storage of two lists with different types. The lists
// should have their types, size, and storage swapped.
TEST_F(VMListTest, SwapStorage) {
- iree_vm_type_def_t element_type_a =
- iree_vm_type_def_make_ref_type(test_a_type_id());
+ iree_vm_type_def_t element_type_a = iree_vm_make_ref_type_def(test_a_type());
iree_vm_list_t* list_a = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type_a, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type_a, /*initial_capacity=*/8,
iree_allocator_system(), &list_a));
iree_host_size_t list_a_size = 4;
for (iree_host_size_t i = 0; i < list_a_size; ++i) {
@@ -702,10 +690,9 @@
IREE_ASSERT_OK(iree_vm_list_push_ref_move(list_a, &ref_a));
}
- iree_vm_type_def_t element_type_b =
- iree_vm_type_def_make_ref_type(test_b_type_id());
+ iree_vm_type_def_t element_type_b = iree_vm_make_ref_type_def(test_b_type());
iree_vm_list_t* list_b = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type_b, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type_b, /*initial_capacity=*/8,
iree_allocator_system(), &list_b));
iree_host_size_t list_b_size = 3;
for (iree_host_size_t i = 0; i < list_b_size; ++i) {
@@ -717,8 +704,7 @@
// list_a should have b types.
iree_vm_type_def_t queried_type_a = iree_vm_list_element_type(list_a);
- EXPECT_EQ(0,
- memcmp(&queried_type_a, &element_type_b, sizeof(element_type_b)));
+ EXPECT_TRUE(iree_vm_type_def_equal(queried_type_a, element_type_b));
EXPECT_EQ(iree_vm_list_size(list_a), list_b_size);
for (iree_host_size_t i = 0; i < list_b_size; ++i) {
iree_vm_ref_t ref_b{0};
@@ -731,8 +717,7 @@
// list_b should have a types.
iree_vm_type_def_t queried_type_b = iree_vm_list_element_type(list_b);
- EXPECT_EQ(0,
- memcmp(&queried_type_b, &element_type_a, sizeof(element_type_a)));
+ EXPECT_TRUE(iree_vm_type_def_equal(queried_type_b, element_type_a));
EXPECT_EQ(iree_vm_list_size(list_b), list_a_size);
for (iree_host_size_t i = 0; i < list_b_size; ++i) {
iree_vm_ref_t ref_a{0};
@@ -751,12 +736,12 @@
// All of the logic for this is shared across all the various copy modes.
TEST_F(VMListTest, CopyOutOfRange) {
iree_vm_type_def_t element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
iree_vm_list_t* src_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, /*initial_capacity=*/8,
iree_allocator_system(), &src_list));
iree_vm_list_t* dst_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, /*initial_capacity=*/8,
iree_allocator_system(), &dst_list));
// Lists are both empty - everything should fail.
@@ -822,11 +807,11 @@
// Tests copying values between lists.
TEST_F(VMListTest, CopyValues) {
iree_vm_type_def_t element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
// src: [0, 1, 2, 3]
iree_vm_list_t* src_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, /*initial_capacity=*/8,
iree_allocator_system(), &src_list));
IREE_ASSERT_OK(iree_vm_list_resize(src_list, 4));
for (iree_host_size_t i = 0; i < 4; ++i) {
@@ -836,7 +821,7 @@
// dst: [4, 5, 6, 7]
iree_vm_list_t* dst_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, /*initial_capacity=*/8,
iree_allocator_system(), &dst_list));
IREE_ASSERT_OK(iree_vm_list_resize(dst_list, 4));
for (iree_host_size_t i = 0; i < 4; ++i) {
@@ -882,9 +867,9 @@
TEST_F(VMListTest, CopyWrongValues) {
// src: [0, 1, 2, 3]
iree_vm_type_def_t src_element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
iree_vm_list_t* src_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&src_element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(src_element_type, /*initial_capacity=*/8,
iree_allocator_system(), &src_list));
IREE_ASSERT_OK(iree_vm_list_resize(src_list, 4));
for (iree_host_size_t i = 0; i < 4; ++i) {
@@ -894,9 +879,9 @@
// dst: [4.0, 5.0, 6.0, 7.0]
iree_vm_type_def_t dst_element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_F32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_F32);
iree_vm_list_t* dst_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&dst_element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(dst_element_type, /*initial_capacity=*/8,
iree_allocator_system(), &dst_list));
IREE_ASSERT_OK(iree_vm_list_resize(dst_list, 4));
for (iree_host_size_t i = 0; i < 4; ++i) {
@@ -914,11 +899,11 @@
// Tests copying refs between lists of !vm.ref<?>.
TEST_F(VMListTest, CopyRefs) {
iree_vm_type_def_t element_type =
- iree_vm_type_def_make_ref_type(IREE_VM_REF_TYPE_ANY);
+ iree_vm_make_ref_type_def(IREE_VM_REF_TYPE_ANY);
// src: [0, 1, 2, 3]
iree_vm_list_t* src_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, /*initial_capacity=*/8,
iree_allocator_system(), &src_list));
iree_host_size_t src_list_size = 4;
for (iree_host_size_t i = 0; i < src_list_size; ++i) {
@@ -928,7 +913,7 @@
// dst: [4, 5, 6, 7]
iree_vm_list_t* dst_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, /*initial_capacity=*/8,
iree_allocator_system(), &dst_list));
iree_host_size_t dst_list_size = 4;
for (iree_host_size_t i = 0; i < dst_list_size; ++i) {
@@ -972,9 +957,9 @@
TEST_F(VMListTest, CopyWrongRefs) {
// src: type A
iree_vm_type_def_t src_element_type =
- iree_vm_type_def_make_ref_type(test_a_type_id());
+ iree_vm_make_ref_type_def(test_a_type());
iree_vm_list_t* src_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&src_element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(src_element_type, /*initial_capacity=*/8,
iree_allocator_system(), &src_list));
iree_host_size_t src_list_size = 4;
for (iree_host_size_t i = 0; i < src_list_size; ++i) {
@@ -984,9 +969,9 @@
// dst: type B
iree_vm_type_def_t dst_element_type =
- iree_vm_type_def_make_ref_type(test_b_type_id());
+ iree_vm_make_ref_type_def(test_b_type());
iree_vm_list_t* dst_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&dst_element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(dst_element_type, /*initial_capacity=*/8,
iree_allocator_system(), &dst_list));
iree_host_size_t dst_list_size = 4;
for (iree_host_size_t i = 0; i < dst_list_size; ++i) {
@@ -1011,7 +996,7 @@
TEST_F(VMListTest, CopyVariants) {
// src: [0, 1, B(2), B(3)]
iree_vm_list_t* src_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(/*element_type=*/nullptr,
+ IREE_ASSERT_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(),
/*initial_capacity=*/8,
iree_allocator_system(), &src_list));
IREE_ASSERT_OK(iree_vm_list_resize(src_list, 4));
@@ -1026,7 +1011,7 @@
// dst: [4, 5, B(6), B(7)]
iree_vm_list_t* dst_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(/*element_type=*/nullptr,
+ IREE_ASSERT_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(),
/*initial_capacity=*/8,
iree_allocator_system(), &dst_list));
IREE_ASSERT_OK(iree_vm_list_resize(dst_list, 4));
@@ -1075,7 +1060,7 @@
TEST_F(VMListTest, CopyFromVariants) {
// src: [0, 1, B(2), B(3)]
iree_vm_list_t* src_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(/*element_type=*/nullptr,
+ IREE_ASSERT_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(),
/*initial_capacity=*/8,
iree_allocator_system(), &src_list));
IREE_ASSERT_OK(iree_vm_list_resize(src_list, 4));
@@ -1090,9 +1075,9 @@
// dst: [4, 5, 6, 7]
iree_vm_type_def_t dst_element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
iree_vm_list_t* dst_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&dst_element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(dst_element_type, /*initial_capacity=*/8,
iree_allocator_system(), &dst_list));
IREE_ASSERT_OK(iree_vm_list_resize(dst_list, 4));
for (iree_host_size_t i = 0; i < 4; ++i) {
@@ -1140,9 +1125,9 @@
TEST_F(VMListTest, CopyToVariants) {
// src: [0, 1, 2, 3]
iree_vm_type_def_t src_element_type =
- iree_vm_type_def_make_value_type(IREE_VM_VALUE_TYPE_I32);
+ iree_vm_make_value_type_def(IREE_VM_VALUE_TYPE_I32);
iree_vm_list_t* src_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&src_element_type, /*initial_capacity=*/8,
+ IREE_ASSERT_OK(iree_vm_list_create(src_element_type, /*initial_capacity=*/8,
iree_allocator_system(), &src_list));
IREE_ASSERT_OK(iree_vm_list_resize(src_list, 4));
for (iree_host_size_t i = 0; i < 4; ++i) {
@@ -1152,7 +1137,7 @@
// dst: [4, 5, B(6), B(7)]
iree_vm_list_t* dst_list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(/*element_type=*/nullptr,
+ IREE_ASSERT_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(),
/*initial_capacity=*/8,
iree_allocator_system(), &dst_list));
IREE_ASSERT_OK(iree_vm_list_resize(dst_list, 4));
@@ -1205,11 +1190,10 @@
// Tests pushing and popping ref objects.
TEST_F(VMListTest, PushPopRef) {
- iree_vm_type_def_t element_type =
- iree_vm_type_def_make_ref_type(test_a_type_id());
+ iree_vm_type_def_t element_type = iree_vm_make_ref_type_def(test_a_type());
iree_host_size_t initial_capacity = 4;
iree_vm_list_t* list = nullptr;
- IREE_ASSERT_OK(iree_vm_list_create(&element_type, initial_capacity,
+ IREE_ASSERT_OK(iree_vm_list_create(element_type, initial_capacity,
iree_allocator_system(), &list));
EXPECT_LE(initial_capacity, iree_vm_list_capacity(list));
EXPECT_EQ(0, iree_vm_list_size(list));
diff --git a/runtime/src/iree/vm/native_module.c b/runtime/src/iree/vm/native_module.c
index 95084b7..1fdcd79 100644
--- a/runtime/src/iree/vm/native_module.c
+++ b/runtime/src/iree/vm/native_module.c
@@ -322,7 +322,7 @@
&module->descriptor->functions[function_ordinal];
iree_status_t status =
function_ptr->shim(stack, flags, args_storage, rets_storage,
- function_ptr->target, module, module_state);
+ function_ptr->target, module->self, module_state);
if (iree_status_is_deferred(status)) {
// Call deferred; bail and return to the scheduler.
// Note that we preserve the stack.
diff --git a/runtime/src/iree/vm/native_module_cc.h b/runtime/src/iree/vm/native_module_cc.h
index 13102ff..35b7ba8 100644
--- a/runtime/src/iree/vm/native_module_cc.h
+++ b/runtime/src/iree/vm/native_module_cc.h
@@ -96,6 +96,8 @@
virtual ~NativeModule() { iree_vm_instance_release(instance_); }
+ iree_vm_instance_t* instance() const { return instance_; }
+
// C API module interface bound to this NativeModule instance.
iree_vm_module_t* interface() { return &interface_; }
diff --git a/runtime/src/iree/vm/native_module_packing.h b/runtime/src/iree/vm/native_module_packing.h
index b828ab5..55319f1 100644
--- a/runtime/src/iree/vm/native_module_packing.h
+++ b/runtime/src/iree/vm/native_module_packing.h
@@ -395,18 +395,18 @@
static void Load(Status& status, params_ptr_t& ptr, storage_type& out_param) {
auto* reg_ptr = reinterpret_cast<iree_vm_ref_t*>(ptr);
ptr += sizeof(iree_vm_ref_t);
- if (reg_ptr->type == ref_type_descriptor<T>::get()->type) {
+ if (reg_ptr->type == ref_type_descriptor<T>::type()) {
out_param = vm::retain_ref(reinterpret_cast<T*>(reg_ptr->ptr));
memset(reg_ptr, 0, sizeof(*reg_ptr));
} else if (IREE_UNLIKELY(reg_ptr->type != IREE_VM_REF_TYPE_NULL)) {
- status =
- iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
- "parameter contains a reference to the wrong type; "
- "have %.*s but expected %.*s",
- (int)iree_vm_ref_type_name(reg_ptr->type).size,
- iree_vm_ref_type_name(reg_ptr->type).data,
- (int)ref_type_descriptor<T>::get()->type_name.size,
- ref_type_descriptor<T>::get()->type_name.data);
+ status = iree_make_status(
+ IREE_STATUS_INVALID_ARGUMENT,
+ "parameter contains a reference to the wrong type; "
+ "have %.*s but expected %.*s",
+ (int)iree_vm_ref_type_name(reg_ptr->type).size,
+ iree_vm_ref_type_name(reg_ptr->type).data,
+ (int)iree_vm_ref_type_name(ref_type_descriptor<T>::type()).size,
+ iree_vm_ref_type_name(ref_type_descriptor<T>::type()).data);
} else {
out_param = {};
}
@@ -420,18 +420,18 @@
static void Load(Status& status, params_ptr_t& ptr, storage_type& out_param) {
auto* reg_ptr = reinterpret_cast<iree_vm_ref_t*>(ptr);
ptr += sizeof(iree_vm_ref_t);
- if (reg_ptr->type == ref_type_descriptor<T>::get()->type) {
+ if (reg_ptr->type == ref_type_descriptor<T>::type()) {
out_param = vm::retain_ref(reinterpret_cast<T*>(reg_ptr->ptr));
memset(reg_ptr, 0, sizeof(*reg_ptr));
} else if (IREE_UNLIKELY(reg_ptr->type != IREE_VM_REF_TYPE_NULL)) {
- status =
- iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
- "parameter contains a reference to the wrong type; "
- "have %.*s but expected %.*s",
- (int)iree_vm_ref_type_name(reg_ptr->type).size,
- iree_vm_ref_type_name(reg_ptr->type).data,
- (int)ref_type_descriptor<T>::get()->type_name.size,
- ref_type_descriptor<T>::get()->type_name.data);
+ status = iree_make_status(
+ IREE_STATUS_INVALID_ARGUMENT,
+ "parameter contains a reference to the wrong type; "
+ "have %.*s but expected %.*s",
+ (int)iree_vm_ref_type_name(reg_ptr->type).size,
+ iree_vm_ref_type_name(reg_ptr->type).data,
+ (int)iree_vm_ref_type_name(ref_type_descriptor<T>::type()).size,
+ iree_vm_ref_type_name(ref_type_descriptor<T>::type()).data);
} else {
out_param = {};
}
@@ -446,7 +446,7 @@
static void Load(Status& status, params_ptr_t& ptr, storage_type& out_param) {
auto* reg_ptr = reinterpret_cast<iree_vm_ref_t*>(ptr);
ptr += sizeof(iree_vm_ref_t);
- if (reg_ptr->type == ref_type_descriptor<iree_vm_buffer_t>::get()->type) {
+ if (reg_ptr->type == ref_type_descriptor<iree_vm_buffer_t>::type()) {
auto byte_span = reinterpret_cast<iree_vm_buffer_t*>(reg_ptr->ptr)->data;
out_param = iree_make_string_view(
reinterpret_cast<const char*>(byte_span.data), byte_span.data_length);
@@ -457,8 +457,11 @@
"have %.*s but expected %.*s",
(int)iree_vm_ref_type_name(reg_ptr->type).size,
iree_vm_ref_type_name(reg_ptr->type).data,
- (int)ref_type_descriptor<iree_vm_buffer_t>::get()->type_name.size,
- ref_type_descriptor<iree_vm_buffer_t>::get()->type_name.data);
+ (int)iree_vm_ref_type_name(
+ ref_type_descriptor<iree_vm_buffer_t>::type())
+ .size,
+ iree_vm_ref_type_name(ref_type_descriptor<iree_vm_buffer_t>::type())
+ .data);
} else {
// NOTE: empty string is allowed here!
out_param = iree_string_view_empty();
@@ -472,7 +475,7 @@
static void Load(Status& status, params_ptr_t& ptr, storage_type& out_param) {
auto* reg_ptr = reinterpret_cast<iree_vm_ref_t*>(ptr);
ptr += sizeof(iree_vm_ref_t);
- if (reg_ptr->type == ref_type_descriptor<iree_vm_buffer_t>::get()->type) {
+ if (reg_ptr->type == ref_type_descriptor<iree_vm_buffer_t>::type()) {
auto byte_span = reinterpret_cast<iree_vm_buffer_t*>(reg_ptr->ptr)->data;
out_param = std::string_view{
reinterpret_cast<const char*>(byte_span.data), byte_span.data_length};
@@ -483,8 +486,11 @@
"have %.*s but expected %.*s",
(int)iree_vm_ref_type_name(reg_ptr->type).size,
iree_vm_ref_type_name(reg_ptr->type).data,
- (int)ref_type_descriptor<iree_vm_buffer_t>::get()->type_name.size,
- ref_type_descriptor<iree_vm_buffer_t>::get()->type_name.data);
+ (int)iree_vm_ref_type_name(
+ ref_type_descriptor<iree_vm_buffer_t>::type())
+ .size,
+ iree_vm_ref_type_name(ref_type_descriptor<iree_vm_buffer_t>::type())
+ .data);
} else {
// NOTE: empty string is allowed here!
out_param = {};
diff --git a/runtime/src/iree/vm/native_module_test.cc b/runtime/src/iree/vm/native_module_test.cc
index e8aa425..24684df 100644
--- a/runtime/src/iree/vm/native_module_test.cc
+++ b/runtime/src/iree/vm/native_module_test.cc
@@ -27,7 +27,8 @@
class VMNativeModuleTest : public ::testing::Test {
protected:
virtual void SetUp() {
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance_));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance_));
// Create both modules shared instances. These are generally immutable and
// can be shared by multiple contexts.
@@ -69,14 +70,16 @@
// Setup I/O lists and pass in the argument. The result list will be
// populated upon return.
vm::ref<iree_vm_list_t> input_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 1, iree_allocator_system(), &input_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 1, iree_allocator_system(),
+ &input_list));
auto arg0_value = iree_vm_value_make_i32(arg0);
IREE_RETURN_IF_ERROR(
iree_vm_list_push_value(input_list.get(), &arg0_value));
vm::ref<iree_vm_list_t> output_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 1, iree_allocator_system(), &output_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 1, iree_allocator_system(),
+ &output_list));
// Invoke the entry function to do our work. Runs synchronously.
IREE_RETURN_IF_ERROR(
diff --git a/runtime/src/iree/vm/native_module_test.h b/runtime/src/iree/vm/native_module_test.h
index 9168a57..6c83622 100644
--- a/runtime/src/iree/vm/native_module_test.h
+++ b/runtime/src/iree/vm/native_module_test.h
@@ -152,7 +152,7 @@
iree_allocator_t allocator;
// Resolved types; these never change once queried and are safe to store on
// the shared structure to avoid needing to look them up again.
- const iree_vm_ref_type_descriptor_t* types[1];
+ iree_vm_ref_type_t types[1];
} module_b_t;
// Stores per-context state; at the minimum imports, but possibly other user
@@ -286,9 +286,12 @@
module->allocator = allocator;
// Resolve types used by the module once so that we can share it across all
- // instances of the module.
- module->types[0] =
- iree_vm_ref_lookup_registered_type(iree_make_cstring_view("vm.buffer"));
+ // instances of the module. Depending on the types here can be somewhat risky
+ // as it can lead to ordering issues. If possible resolving types on module
+ // state is better as all dependent modules are guaranteed to have been
+ // loaded.
+ module->types[0] = iree_vm_instance_lookup_type(
+ instance, iree_make_cstring_view("vm.buffer"));
if (!module->types[0]) {
iree_allocator_free(allocator, module);
return iree_make_status(
diff --git a/runtime/src/iree/vm/ops_emitc.h b/runtime/src/iree/vm/ops_emitc.h
index 60b4062..4ae8c8e 100644
--- a/runtime/src/iree/vm/ops_emitc.h
+++ b/runtime/src/iree/vm/ops_emitc.h
@@ -46,9 +46,15 @@
body \
} typename;
+// Get an array element
+#define EMITC_ARRAY_ELEMENT(array, index) (array)[index]
+
// Get the address of an array element
#define EMITC_ARRAY_ELEMENT_ADDRESS(array, index) &(array)[index]
+// Assign a value to an array at a given index
+#define EMITC_ARRAY_ELEMENT_ASSIGN(array, index, value) (array)[index] = (value)
+
// Unary operators
#define EMITC_UNARY(op, arg) (op(arg))
diff --git a/runtime/src/iree/vm/ref.c b/runtime/src/iree/vm/ref.c
index 5e4c822..e610157 100644
--- a/runtime/src/iree/vm/ref.c
+++ b/runtime/src/iree/vm/ref.c
@@ -10,102 +10,6 @@
#include "iree/base/internal/atomics.h"
-// TODO(benvanik): dynamic, if we care - otherwise keep small.
-// After a dozen or so types the linear scan will likely start to spill the
-// DCACHE and need to be reworked. I suspect at the time we have >=64 types
-// we'll want to rewrite all of this anyway (using externalized type ID storage
-// or something more complex).
-#define IREE_VM_MAX_TYPE_ID 64
-
-static inline volatile iree_atomic_ref_count_t* iree_vm_get_raw_counter_ptr(
- void* ptr, const iree_vm_ref_type_descriptor_t* type_descriptor) {
- return (volatile iree_atomic_ref_count_t*)(((uintptr_t)(ptr)) +
- type_descriptor->offsetof_counter);
-}
-
-static inline volatile iree_atomic_ref_count_t* iree_vm_get_ref_counter_ptr(
- iree_vm_ref_t* ref) {
- return (volatile iree_atomic_ref_count_t*)(((uintptr_t)ref->ptr) +
- ref->offsetof_counter);
-}
-
-IREE_API_EXPORT void iree_vm_ref_object_retain(
- void* ptr, const iree_vm_ref_type_descriptor_t* type_descriptor) {
- if (!ptr) return;
- volatile iree_atomic_ref_count_t* counter =
- iree_vm_get_raw_counter_ptr(ptr, type_descriptor);
- iree_atomic_ref_count_inc(counter);
-}
-
-IREE_API_EXPORT void iree_vm_ref_object_release(
- void* ptr, const iree_vm_ref_type_descriptor_t* type_descriptor) {
- if (!ptr) return;
- volatile iree_atomic_ref_count_t* counter =
- iree_vm_get_raw_counter_ptr(ptr, type_descriptor);
- if (iree_atomic_ref_count_dec(counter) == 1) {
- if (type_descriptor->destroy) {
- // NOTE: this makes us not re-entrant, but I think that's OK.
- type_descriptor->destroy(ptr);
- }
- }
-}
-
-// A table of type descriptors registered at startup.
-// These provide quick dereferencing of destruction functions and type names for
-// debugging. Note that this just points to registered descriptors (or NULL) for
-// each type ID in the type range and does not own the descriptors.
-//
-// Note that [0] is always the NULL type and has a NULL descriptor. We don't
-// allow types to be registered there.
-static const iree_vm_ref_type_descriptor_t*
- iree_vm_ref_type_descriptors[IREE_VM_MAX_TYPE_ID] = {0};
-
-// Returns the type descriptor (or NULL) for the given type ID.
-static const iree_vm_ref_type_descriptor_t* iree_vm_ref_get_type_descriptor(
- iree_vm_ref_type_t type) {
- if (type >= IREE_VM_MAX_TYPE_ID) {
- return NULL;
- }
- return iree_vm_ref_type_descriptors[type];
-}
-
-IREE_API_EXPORT iree_status_t
-iree_vm_ref_register_type(iree_vm_ref_type_descriptor_t* descriptor) {
- for (int i = 1; i <= IREE_VM_MAX_TYPE_ID; ++i) {
- if (!iree_vm_ref_type_descriptors[i]) {
- iree_vm_ref_type_descriptors[i] = descriptor;
- descriptor->type = i;
- return iree_ok_status();
- }
- }
- // Too many user-defined types registered; need to increase
- // IREE_VM_MAX_TYPE_ID.
- return iree_make_status(IREE_STATUS_RESOURCE_EXHAUSTED,
- "too many user-defined types registered; new type "
- "would exceed maximum of %d",
- IREE_VM_MAX_TYPE_ID);
-}
-
-IREE_API_EXPORT iree_string_view_t
-iree_vm_ref_type_name(iree_vm_ref_type_t type) {
- if (type == 0 || type >= IREE_VM_MAX_TYPE_ID) {
- return iree_string_view_empty();
- }
- return iree_vm_ref_type_descriptors[type]->type_name;
-}
-
-IREE_API_EXPORT const iree_vm_ref_type_descriptor_t*
-iree_vm_ref_lookup_registered_type(iree_string_view_t full_name) {
- for (int i = 1; i <= IREE_VM_MAX_TYPE_ID; ++i) {
- if (!iree_vm_ref_type_descriptors[i]) break;
- if (iree_string_view_equal(iree_vm_ref_type_descriptors[i]->type_name,
- full_name)) {
- return iree_vm_ref_type_descriptors[i];
- }
- }
- return NULL;
-}
-
// Useful debugging tool:
#if 0
static void iree_vm_ref_trace(const char* msg, iree_vm_ref_t* ref) {
@@ -114,19 +18,76 @@
fprintf(stderr, "%s %.*s 0x%p %d\n", msg, (int)name.size, name.data, ref->ptr,
counter->__val);
}
+static void iree_vm_ref_ptr_trace(const char* msg, void* ptr,
+ iree_vm_ref_type_t type) {
+ volatile iree_atomic_ref_count_t* counter =
+ iree_vm_get_raw_counter_ptr(ptr, type);
+ iree_string_view_t name = iree_vm_ref_type_name(type);
+ fprintf(stderr, "%s %.*s 0x%p %d\n", msg, (int)name.size, name.data, ptr,
+ counter->__val);
+}
#else
#define iree_vm_ref_trace(...)
+#define iree_vm_ref_ptr_trace(...)
#endif // 0
+IREE_API_EXPORT iree_string_view_t
+iree_vm_ref_type_name(iree_vm_ref_type_t type) {
+ IREE_VM_REF_ASSERT(type);
+ return iree_vm_ref_type_descriptor(type)->type_name;
+}
+
+static inline volatile iree_atomic_ref_count_t* iree_vm_get_raw_counter_ptr(
+ void* ptr, iree_vm_ref_type_t type) {
+ IREE_VM_REF_ASSERT(ptr);
+ IREE_VM_REF_ASSERT(type_descriptor);
+ return (volatile iree_atomic_ref_count_t*)ptr +
+ (type & IREE_VM_REF_TYPE_TAG_BIT_MASK);
+}
+
+static inline volatile iree_atomic_ref_count_t* iree_vm_get_ref_counter_ptr(
+ iree_vm_ref_t* ref) {
+ IREE_VM_REF_ASSERT(ref);
+ IREE_VM_REF_ASSERT(ref->ptr);
+ return (volatile iree_atomic_ref_count_t*)ref->ptr +
+ (ref->type & IREE_VM_REF_TYPE_TAG_BIT_MASK);
+}
+
+IREE_API_EXPORT void iree_vm_ref_object_retain(void* ptr,
+ iree_vm_ref_type_t type) {
+ if (!ptr) return;
+ IREE_VM_REF_ASSERT(type);
+ volatile iree_atomic_ref_count_t* counter =
+ iree_vm_get_raw_counter_ptr(ptr, type);
+ iree_atomic_ref_count_inc(counter);
+ iree_vm_ref_ptr_trace("RETAIN", ptr, type);
+}
+
+IREE_API_EXPORT void iree_vm_ref_object_release(void* ptr,
+ iree_vm_ref_type_t type) {
+ if (!ptr) return;
+ IREE_VM_REF_ASSERT(type);
+ iree_vm_ref_ptr_trace("RELEASE", ptr, type);
+ volatile iree_atomic_ref_count_t* counter =
+ iree_vm_get_raw_counter_ptr(ptr, type);
+ if (iree_atomic_ref_count_dec(counter) == 1) {
+ const iree_vm_ref_type_descriptor_t* descriptor =
+ iree_vm_ref_type_descriptor(type);
+ if (descriptor->destroy) {
+ // NOTE: this makes us not re-entrant, but I think that's OK.
+ iree_vm_ref_ptr_trace("DESTROY", ptr, type);
+ descriptor->destroy(ptr);
+ }
+ }
+}
+
IREE_API_EXPORT iree_status_t iree_vm_ref_wrap_assign(void* ptr,
iree_vm_ref_type_t type,
iree_vm_ref_t* out_ref) {
- const iree_vm_ref_type_descriptor_t* type_descriptor =
- iree_vm_ref_get_type_descriptor(type);
- if (!type_descriptor) {
- return iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
- "type not registered");
- }
+ IREE_VM_REF_ASSERT(ptr);
+ IREE_VM_REF_ASSERT(type);
+ IREE_VM_REF_ASSERT(out_ref);
+ IREE_VM_REF_ASSERT(iree_vm_ref_type_descriptor(type));
if (out_ref->ptr != NULL && out_ref->ptr != ptr) {
// Release existing value.
@@ -136,7 +97,6 @@
// NOTE: we do not manipulate the counter here as we assume it starts at 1
// or it's already coming in with some references.
out_ref->ptr = ptr;
- out_ref->offsetof_counter = type_descriptor->offsetof_counter;
out_ref->type = type;
iree_vm_ref_trace("WRAP ASSIGN", out_ref);
@@ -146,6 +106,9 @@
IREE_API_EXPORT iree_status_t iree_vm_ref_wrap_retain(void* ptr,
iree_vm_ref_type_t type,
iree_vm_ref_t* out_ref) {
+ IREE_VM_REF_ASSERT(ptr);
+ IREE_VM_REF_ASSERT(type);
+ IREE_VM_REF_ASSERT(out_ref);
IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_assign(ptr, type, out_ref));
if (out_ref->ptr) {
volatile iree_atomic_ref_count_t* counter =
@@ -157,6 +120,7 @@
}
IREE_API_EXPORT void iree_vm_ref_retain_inplace(iree_vm_ref_t* ref) {
+ IREE_VM_REF_ASSERT(ref);
if (ref->ptr) {
volatile iree_atomic_ref_count_t* counter =
iree_vm_get_ref_counter_ptr(ref);
@@ -169,6 +133,8 @@
iree_vm_ref_t* out_ref) {
// NOTE: ref and out_ref may alias or be nested so we retain before we
// potentially release.
+ IREE_VM_REF_ASSERT(ref);
+ IREE_VM_REF_ASSERT(out_ref);
iree_vm_ref_t temp_ref = *ref;
if (ref->ptr) {
volatile iree_atomic_ref_count_t* counter =
@@ -187,6 +153,9 @@
IREE_API_EXPORT iree_status_t iree_vm_ref_retain_checked(
iree_vm_ref_t* ref, iree_vm_ref_type_t type, iree_vm_ref_t* out_ref) {
+ IREE_VM_REF_ASSERT(ref);
+ IREE_VM_REF_ASSERT(type);
+ IREE_VM_REF_ASSERT(out_ref);
if (ref->type != IREE_VM_REF_TYPE_NULL && ref->type != type &&
type != IREE_VM_REF_TYPE_ANY) {
return iree_make_status(IREE_STATUS_INVALID_ARGUMENT,
@@ -198,6 +167,8 @@
IREE_API_EXPORT void iree_vm_ref_retain_or_move(int is_move, iree_vm_ref_t* ref,
iree_vm_ref_t* out_ref) {
+ IREE_VM_REF_ASSERT(ref);
+ IREE_VM_REF_ASSERT(out_ref);
if (is_move) {
iree_vm_ref_move(ref, out_ref);
} else {
@@ -208,6 +179,9 @@
IREE_API_EXPORT iree_status_t iree_vm_ref_retain_or_move_checked(
int is_move, iree_vm_ref_t* ref, iree_vm_ref_type_t type,
iree_vm_ref_t* out_ref) {
+ IREE_VM_REF_ASSERT(ref);
+ IREE_VM_REF_ASSERT(type);
+ IREE_VM_REF_ASSERT(out_ref);
if (ref->type != IREE_VM_REF_TYPE_NULL && ref->type != type &&
type != IREE_VM_REF_TYPE_ANY) {
// Make no changes on failure.
@@ -219,17 +193,18 @@
}
IREE_API_EXPORT void iree_vm_ref_release(iree_vm_ref_t* ref) {
+ IREE_VM_REF_ASSERT(ref);
if (ref->type == IREE_VM_REF_TYPE_NULL || ref->ptr == NULL) return;
iree_vm_ref_trace("RELEASE", ref);
volatile iree_atomic_ref_count_t* counter = iree_vm_get_ref_counter_ptr(ref);
if (iree_atomic_ref_count_dec(counter) == 1) {
- const iree_vm_ref_type_descriptor_t* type_descriptor =
- iree_vm_ref_get_type_descriptor(ref->type);
- if (type_descriptor->destroy) {
+ const iree_vm_ref_type_descriptor_t* descriptor =
+ iree_vm_ref_type_descriptor(ref->type);
+ if (descriptor->destroy) {
// NOTE: this makes us not re-entrant, but I think that's OK.
iree_vm_ref_trace("DESTROY", ref);
- type_descriptor->destroy(ref->ptr);
+ descriptor->destroy(ref->ptr);
}
}
@@ -239,6 +214,9 @@
IREE_API_EXPORT void iree_vm_ref_assign(iree_vm_ref_t* ref,
iree_vm_ref_t* out_ref) {
+ IREE_VM_REF_ASSERT(ref);
+ IREE_VM_REF_ASSERT(out_ref);
+
// NOTE: ref and out_ref may alias.
iree_vm_ref_t temp_ref = *ref;
if (ref == out_ref) {
@@ -255,6 +233,9 @@
IREE_API_EXPORT void iree_vm_ref_move(iree_vm_ref_t* ref,
iree_vm_ref_t* out_ref) {
+ IREE_VM_REF_ASSERT(ref);
+ IREE_VM_REF_ASSERT(out_ref);
+
// NOTE: ref and out_ref may alias.
if (ref == out_ref) {
// Source == target; ignore entirely.
@@ -275,6 +256,7 @@
}
IREE_API_EXPORT bool iree_vm_ref_is_null(const iree_vm_ref_t* ref) {
+ IREE_VM_REF_ASSERT(ref);
return ref->type == IREE_VM_REF_TYPE_NULL;
}
diff --git a/runtime/src/iree/vm/ref.h b/runtime/src/iree/vm/ref.h
index c2d1678..19d0c25 100644
--- a/runtime/src/iree/vm/ref.h
+++ b/runtime/src/iree/vm/ref.h
@@ -19,6 +19,29 @@
extern "C" {
#endif // __cplusplus
+typedef struct iree_vm_instance_t iree_vm_instance_t;
+
+// Enable additional expensive checks to track down ref counting issues.
+// #define IREE_VM_REF_PARANOID
+
+// Number of least significant bits available in an iree_vm_ref_type_t value.
+// These can be used for any purpose.
+#define IREE_VM_REF_TYPE_TAG_BITS 3
+#define IREE_VM_REF_TYPE_TAG_BIT_MASK 0b111
+
+// Number of bits available in an iree_vm_ref_type_t value to hold the pointer.
+#define IREE_VM_REF_TYPE_PTR_BITS \
+ (sizeof(uintptr_t) * 8 - IREE_VM_REF_TYPE_TAG_BITS)
+#define IREE_VM_REF_TYPE_PTR_BIT_MASK (~IREE_VM_REF_TYPE_TAG_BIT_MASK)
+
+// (1 << IREE_VM_REF_TYPE_TAG_BITS) hardcoded for MSVC which cannot have
+// expressions in its alignas in older versions.
+#define IREE_VM_REF_TYPE_DESCRIPTOR_ALIGNMENT 8
+
+// Alignment required for the reference counter.
+// Used to pack the offset bits to fit in IREE_VM_REF_TYPE_TAG_BITS.
+#define IREE_VM_REF_COUNTER_ALIGNMENT sizeof(iree_atomic_ref_count_t)
+
// Defines the type of the reference-counted pointer.
// This is used to verify that operations dealing with the variant ref struct
// are correct at runtime. We don't allow control over the ref types from the
@@ -30,14 +53,49 @@
// NOTE: these type values are assigned dynamically right now. Treat them as
// opaque and unstable across process invocations.
- // Maximum type ID value. Type IDs are limited to 24-bits.
- IREE_VM_REF_TYPE_MAX_VALUE = 0x00FFFFFEu,
-
// Wildcard type that indicates that a value may be a ref type but of an
- // unspecified internal type.
- IREE_VM_REF_TYPE_ANY = 0x00FFFFFFu,
+ // unspecified internal type. Note that we mask off the bottom bits to allow
+ // for tagging (all ref type values )
+ IREE_VM_REF_TYPE_ANY = UINTPTR_MAX ^ IREE_VM_REF_TYPE_TAG_BIT_MASK,
};
-typedef uint32_t iree_vm_ref_type_t;
+
+typedef void(IREE_API_PTR* iree_vm_ref_destroy_t)(void* ptr);
+
+// Describes a type for the VM.
+typedef iree_alignas(IREE_VM_REF_TYPE_DESCRIPTOR_ALIGNMENT) struct
+ iree_vm_ref_type_descriptor_t {
+ // Function called when references of this type reach 0 and should be
+ // destroyed.
+ iree_vm_ref_destroy_t destroy;
+ // Unretained type name that can be used for debugging.
+ iree_string_view_t type_name;
+ // Offset from ptr in units of IREE_VM_REF_COUNTER_ALIGNMENT to the start of
+ // an iree_atomic_ref_count_t representing the current reference count.
+ uintptr_t offsetof_counter : IREE_VM_REF_TYPE_TAG_BITS;
+ uintptr_t reserved : IREE_VM_REF_TYPE_PTR_BITS;
+} iree_vm_ref_type_descriptor_t;
+
+// Type-erased reference counted type descriptor.
+typedef uintptr_t iree_vm_ref_type_t;
+
+// Makes an opaque reference to a type descriptor.
+static inline iree_vm_ref_type_t iree_vm_make_ref_type(
+ const iree_vm_ref_type_descriptor_t* descriptor) {
+ return (iree_vm_ref_type_t)descriptor |
+ (iree_vm_ref_type_t)descriptor->offsetof_counter;
+}
+
+// Returns the type name for the given type, if found.
+IREE_API_EXPORT iree_string_view_t
+iree_vm_ref_type_name(iree_vm_ref_type_t type);
+
+// Returns the type descriptor backing the given |type|.
+// The descriptor is an implementation detail.
+static inline const iree_vm_ref_type_descriptor_t* iree_vm_ref_type_descriptor(
+ iree_vm_ref_type_t type) {
+ return (const iree_vm_ref_type_descriptor_t*)(type &
+ IREE_VM_REF_TYPE_PTR_BIT_MASK);
+}
// Base for iree_vm_ref_t object targets.
//
@@ -52,8 +110,8 @@
// static iree_vm_ref_type_descriptor_t my_type_descriptor;
// my_type_descriptor.type_name = iree_string_view_t{"my_type", 7};
// my_type_descriptor.destroy = my_type_destroy;
-// my_type_descriptor.offsetof_counter = offsetof(my_type_t,
-// ref_object.counter);
+// my_type_descriptor.offsetof_counter =
+// offsetof(my_type_t, ref_object.counter) / IREE_VM_REF_COUNTER_ALIGNMENT;
// iree_vm_ref_register_defined_type(&my_type_descriptor);
//
// Usage (C++):
@@ -76,71 +134,28 @@
// Pointer to the object. Type is resolved based on the |type| field.
// Will be NULL if the reference points to nothing.
void* ptr;
- // Offset from ptr, in bytes, to the start of an atomic_int32_t representing
- // the current reference count. We store this here to avoid the need for an
- // indirection in the (extremely common) case of just reference count inc/dec.
- uint32_t offsetof_counter : 8;
// Registered type of the object pointed to by ptr.
- iree_vm_ref_type_t type : 24;
+ iree_vm_ref_type_t type;
} iree_vm_ref_t;
static_assert(
sizeof(iree_vm_ref_t) <= sizeof(void*) * 2,
"iree_vm_ref_t dominates stack space usage and should be kept tiny");
-typedef void(IREE_API_PTR* iree_vm_ref_destroy_t)(void* ptr);
-
-// Describes a type for the VM.
-typedef struct iree_vm_ref_type_descriptor_t {
- // Function called when references of this type reach 0 and should be
- // destroyed.
- iree_vm_ref_destroy_t destroy;
- // Offset from ptr, in bytes, to the start of an atomic_int32_t representing
- // the current reference count.
- uint32_t offsetof_counter : 8;
- // The type ID assigned to this type from the iree_vm_ref_type_t table (or an
- // external user source).
- iree_vm_ref_type_t type : 24;
- // Unretained type name that can be used for debugging.
- iree_string_view_t type_name;
-} iree_vm_ref_type_descriptor_t;
-
-// Directly retains the object with base |ptr| with the given |type_descriptor|.
+// Directly retains the object with base |ptr| with the given |type|.
//
// Note that this avoids any kind of type checking; for untrusted inputs use
// the iree_vm_ref_t-based methods.
-IREE_API_EXPORT void iree_vm_ref_object_retain(
- void* ptr, const iree_vm_ref_type_descriptor_t* type_descriptor);
+IREE_API_EXPORT void iree_vm_ref_object_retain(void* ptr,
+ iree_vm_ref_type_t type);
-// Directly release the object with base |ptr| with the given |type_descriptor|,
+// Directly release the object with base |ptr| with the given |type|,
// possibly destroying it if it is the last reference. Assume that |ptr| is
// invalid after this function returns.
//
// Note that this avoids any kind of type checking; for untrusted inputs use
// the iree_vm_ref_t-based methods.
-IREE_API_EXPORT void iree_vm_ref_object_release(
- void* ptr, const iree_vm_ref_type_descriptor_t* type_descriptor);
-
-// Registers a user-defined type with the IREE C ref system.
-// The provided destroy function will be used to destroy objects when their
-// reference count goes to 0. NULL can be used to no-op the destruction if the
-// type is not owned by the VM.
-//
-// TODO(benvanik): keep names alive for user types?
-// NOTE: the name is not retained and must be kept live by the caller. Ideally
-// it is stored in static read-only memory in the binary.
-//
-// WARNING: this function is not thread-safe and should only be used at startup
-// to register the types. Do not call this while any refs may be alive.
-IREE_API_EXPORT iree_status_t
-iree_vm_ref_register_type(iree_vm_ref_type_descriptor_t* descriptor);
-
-// Returns the type name for the given type, if found.
-IREE_API_EXPORT iree_string_view_t
-iree_vm_ref_type_name(iree_vm_ref_type_t type);
-
-// Returns the registered type descriptor for the given type, if found.
-IREE_API_EXPORT const iree_vm_ref_type_descriptor_t*
-iree_vm_ref_lookup_registered_type(iree_string_view_t full_name);
+IREE_API_EXPORT void iree_vm_ref_object_release(void* ptr,
+ iree_vm_ref_type_t type);
// Returns a NULL ref wrapper.
static inline iree_vm_ref_t iree_vm_ref_null(void) {
@@ -237,18 +252,27 @@
// Type adapter utilities for interfacing with the VM
//===----------------------------------------------------------------------===//
+#if defined(IREE_VM_REF_PARANOID)
+#define IREE_VM_REF_ASSERT(expr) IREE_ASSERT(expr)
+#else
+#define IREE_VM_REF_ASSERT(expr)
+#endif // IREE_VM_REF_PARANOID
+
// TODO(benvanik): make these macros standard/document them.
#define IREE_VM_DECLARE_TYPE_ADAPTERS(name, T) \
- IREE_API_EXPORT_VARIABLE iree_vm_ref_type_descriptor_t name##_descriptor; \
- static inline iree_vm_ref_type_t name##_type_id() { \
- return name##_descriptor.type; \
+ IREE_API_EXPORT_VARIABLE iree_vm_ref_type_t name##_registration; \
+ static inline iree_vm_ref_type_t name##_type(void) { \
+ IREE_VM_REF_ASSERT(name##_registration); \
+ return name##_registration; \
} \
static inline bool name##_isa(const iree_vm_ref_t ref) { \
- return name##_descriptor.type == ref.type; \
+ IREE_VM_REF_ASSERT(name##_registration); \
+ return name##_registration == ref.type; \
} \
IREE_API_EXPORT iree_vm_ref_t name##_retain_ref(T* value); \
IREE_API_EXPORT iree_vm_ref_t name##_move_ref(T* value); \
static inline T* name##_deref(const iree_vm_ref_t ref) { \
+ IREE_VM_REF_ASSERT(name##_registration); \
return IREE_LIKELY(name##_isa(ref)) ? (T*)ref.ptr : NULL; \
} \
IREE_API_EXPORT iree_status_t name##_check_deref(const iree_vm_ref_t ref, \
@@ -259,27 +283,31 @@
// TODO(benvanik): make these macros standard/document them.
#define IREE_VM_DEFINE_TYPE_ADAPTERS(name, T) \
- iree_vm_ref_type_descriptor_t name##_descriptor = {0}; \
+ iree_vm_ref_type_t name##_registration = 0; \
IREE_API_EXPORT iree_vm_ref_t name##_retain_ref(T* value) { \
+ IREE_VM_REF_ASSERT(name##_registration); \
iree_vm_ref_t ref = {0}; \
- iree_vm_ref_wrap_retain(value, name##_descriptor.type, &ref); \
+ iree_vm_ref_wrap_retain(value, name##_type(), &ref); \
return ref; \
} \
IREE_API_EXPORT iree_vm_ref_t name##_move_ref(T* value) { \
+ IREE_VM_REF_ASSERT(name##_registration); \
iree_vm_ref_t ref = {0}; \
- iree_vm_ref_wrap_assign(value, name##_descriptor.type, &ref); \
+ iree_vm_ref_wrap_assign(value, name##_type(), &ref); \
return ref; \
} \
IREE_API_EXPORT iree_status_t name##_check_deref(const iree_vm_ref_t ref, \
T** out_ptr) { \
- IREE_RETURN_IF_ERROR(iree_vm_ref_check(ref, name##_descriptor.type)); \
+ IREE_VM_REF_ASSERT(name##_registration); \
+ IREE_RETURN_IF_ERROR(iree_vm_ref_check(ref, name##_type())); \
*out_ptr = (T*)ref.ptr; \
return iree_ok_status(); \
} \
IREE_API_EXPORT iree_status_t name##_check_deref_or_null( \
const iree_vm_ref_t ref, T** out_ptr) { \
+ IREE_VM_REF_ASSERT(name##_registration); \
if (ref.type != IREE_VM_REF_TYPE_NULL) { \
- IREE_RETURN_IF_ERROR(iree_vm_ref_check(ref, name##_descriptor.type)); \
+ IREE_RETURN_IF_ERROR(iree_vm_ref_check(ref, name##_type())); \
*out_ptr = (T*)ref.ptr; \
} else { \
*out_ptr = NULL; \
@@ -292,7 +320,6 @@
#include "iree/vm/ref_cc.h"
#else
#define IREE_VM_DECLARE_CC_TYPE_LOOKUP(name, T)
-#define IREE_VM_REGISTER_CC_TYPE(type, name, descriptor)
#define IREE_VM_DECLARE_CC_TYPE_ADAPTERS(name, T)
#endif // __cplusplus
diff --git a/runtime/src/iree/vm/ref_cc.h b/runtime/src/iree/vm/ref_cc.h
index e630956..3240e48 100644
--- a/runtime/src/iree/vm/ref_cc.h
+++ b/runtime/src/iree/vm/ref_cc.h
@@ -33,19 +33,19 @@
template <typename T>
struct ref_type_descriptor {
- static const iree_vm_ref_type_descriptor_t* get();
+ static iree_vm_ref_type_t type();
};
// Users may override this with their custom types to allow the packing code to
// access their registered type ID at runtime.
template <typename T>
static inline void ref_type_retain(T* p) {
- iree_vm_ref_object_retain(p, ref_type_descriptor<T>::get());
+ iree_vm_ref_object_retain(p, ref_type_descriptor<T>::type());
}
template <typename T>
static inline void ref_type_release(T* p) {
- iree_vm_ref_object_release(p, ref_type_descriptor<T>::get());
+ iree_vm_ref_object_release(p, ref_type_descriptor<T>::type());
}
// Base class for reference counted objects.
@@ -163,7 +163,9 @@
// If the type has virtual methods (dtors/etc) then it should be 4 or 8
// (depending on pointer width). It may be other things, and instead of too
// much crazy magic we just rely on offsetof doing the right thing here.
- static constexpr size_t offsetof_counter() { return offsetof(T, counter_); }
+ static constexpr size_t offsetof_counter() {
+ return offsetof(T, counter_) / IREE_VM_REF_COUNTER_ALIGNMENT;
+ }
protected:
RefObject() { ref_ptr_add_ref(static_cast<T*>(this)); }
@@ -222,11 +224,11 @@
// Compatible only with types that implement the following methods:
// ref_type_retain(T*)
// ref_type_release(T*)
-// ref_type_descriptor<T>::get()
+// ref_type_descriptor<T>::type()
//
-// If you get link errors pertaining to ref_type_descriptor then ensure that you
-// have included the header file containing the IREE_VM_DECLARE_TYPE_ADAPTERS
-// for the given type.
+// If you get link errors pertaining to ref_type_registration then ensure that
+// you have included the header file containing the
+// IREE_VM_DECLARE_TYPE_ADAPTERS for the given type.
//
// TODO(benvanik): reconcile RefObject, iree_vm_ref_t, and this.
template <typename T>
@@ -236,27 +238,20 @@
public:
IREE_ATTRIBUTE_ALWAYS_INLINE iree_vm_ref_type_t type() const noexcept {
- return ref_type_descriptor<T>::get()->type;
+ IREE_VM_REF_ASSERT(ref_type_descriptor<T>::type());
+ return ref_type_descriptor<T>::type();
}
- IREE_ATTRIBUTE_ALWAYS_INLINE ref() noexcept
- : ref_({
- 0,
- ref_type_descriptor<T>::get()->offsetof_counter,
- ref_type_descriptor<T>::get()->type,
- }) {}
- IREE_ATTRIBUTE_ALWAYS_INLINE ref(std::nullptr_t) noexcept // NOLINT
- : ref_({
- 0,
- ref_type_descriptor<T>::get()->offsetof_counter,
- ref_type_descriptor<T>::get()->type,
- }) {}
- IREE_ATTRIBUTE_ALWAYS_INLINE ref(T* p) noexcept // NOLINT
- : ref_({
- p,
- ref_type_descriptor<T>::get()->offsetof_counter,
- ref_type_descriptor<T>::get()->type,
- }) {}
+ IREE_ATTRIBUTE_ALWAYS_INLINE ref() noexcept : ref_(iree_vm_ref_null()) {}
+ IREE_ATTRIBUTE_ALWAYS_INLINE ref(std::nullptr_t) noexcept {}
+ IREE_ATTRIBUTE_ALWAYS_INLINE ref(T* p) noexcept {
+ if (!p) return;
+ ref_.ptr = p;
+ ref_.type = ref_type_descriptor<T>::type();
+ }
+ // TODO(benvanik): use the offsetof_counter we already have locally here and
+ // below. In theory the compiler may be able to optimize some of this based on
+ // pointer equality but investigation is required.
IREE_ATTRIBUTE_ALWAYS_INLINE ~ref() noexcept { ref_type_release<T>(get()); }
// Don't use implicit ref copying; use retain_ref instead to make things more
@@ -280,7 +275,6 @@
template <typename U>
ref(ref<U>&& rhs) noexcept { // NOLINT
ref_.ptr = static_cast<T*>(rhs.release());
- ref_.offsetof_counter = rhs.ref_.offsetof_counter;
ref_.type = rhs.ref_.type;
}
template <typename U>
@@ -296,7 +290,7 @@
// deleting it.
void reset() noexcept {
ref_type_release<T>(get());
- ref_.ptr = nullptr;
+ ref_ = iree_vm_ref_null();
}
// Releases a pointer.
@@ -306,7 +300,7 @@
// To re-wrap in a ref use either ref<T>(value) or assign().
IREE_ATTRIBUTE_ALWAYS_INLINE T* release() noexcept {
T* p = get();
- ref_.ptr = nullptr;
+ ref_ = iree_vm_ref_null();
return p;
}
@@ -316,6 +310,7 @@
IREE_ATTRIBUTE_ALWAYS_INLINE void assign(T* value) noexcept {
reset();
ref_.ptr = value;
+ ref_.type = ref_type_descriptor<T>::type();
}
// Gets the pointer referenced by this instance.
@@ -340,7 +335,7 @@
constexpr bool operator!() const noexcept { return !get(); }
// Swap support.
- void swap(ref& rhs) { std::swap(ref_.ptr, rhs.ref_.ptr); }
+ void swap(ref& rhs) { std::swap(ref_, rhs.ref_); }
// Allows directly passing the ref to a C-API function for creation.
// Example:
@@ -351,7 +346,7 @@
}
private:
- mutable iree_vm_ref_t ref_;
+ mutable iree_vm_ref_t ref_ = {0};
};
// Constructs an object of type T and wraps it in a reference.
@@ -482,24 +477,19 @@
// dynamic type registration mechanism and that can be wrapped in an
// iree_vm_ref_t.
-#define IREE_VM_DECLARE_CC_TYPE_LOOKUP(name, T) \
- namespace iree { \
- namespace vm { \
- template <> \
- struct ref_type_descriptor<T> { \
- static inline const iree_vm_ref_type_descriptor_t* get() { \
- return &name##_descriptor; \
- } \
- }; \
- } \
+#define IREE_VM_DECLARE_CC_TYPE_LOOKUP(name, T) \
+ namespace iree { \
+ namespace vm { \
+ template <> \
+ struct ref_type_descriptor<T> { \
+ static inline const iree_vm_ref_type_descriptor_t* get() { \
+ return reinterpret_cast<iree_vm_ref_type_descriptor_t*>(name##_type()); \
+ } \
+ static inline iree_vm_ref_type_t type() { return name##_type(); } \
+ }; \
+ } \
}
-#define IREE_VM_REGISTER_CC_TYPE(type, name, descriptor) \
- descriptor.type_name = iree_make_cstring_view(name); \
- descriptor.offsetof_counter = type::offsetof_counter(); \
- descriptor.destroy = type::DirectDestroy; \
- IREE_RETURN_IF_ERROR(iree_vm_ref_register_type(&descriptor));
-
//===----------------------------------------------------------------------===//
// ref-type registration and declaration for core VM types
//===----------------------------------------------------------------------===//
@@ -527,7 +517,7 @@
IREE_ATTRIBUTE_ALWAYS_INLINE ref() noexcept : ptr_(nullptr) {} \
IREE_ATTRIBUTE_ALWAYS_INLINE ref(std::nullptr_t) noexcept \
: ptr_(nullptr) {} \
- IREE_ATTRIBUTE_ALWAYS_INLINE ref(T* p) noexcept : ptr_(nullptr) {} \
+ IREE_ATTRIBUTE_ALWAYS_INLINE ref(T* p) noexcept : ptr_(p) {} \
IREE_ATTRIBUTE_ALWAYS_INLINE ~ref() noexcept { \
ref_type_release<T>(get()); \
} \
diff --git a/runtime/src/iree/vm/ref_test.cc b/runtime/src/iree/vm/ref_test.cc
index 57b4b95..ce4b932 100644
--- a/runtime/src/iree/vm/ref_test.cc
+++ b/runtime/src/iree/vm/ref_test.cc
@@ -11,10 +11,20 @@
#include "iree/base/api.h"
#include "iree/testing/gtest.h"
#include "iree/testing/status_matchers.h"
+#include "iree/vm/instance.h"
#include "iree/vm/ref.h"
namespace {
+using InstancePtr =
+ std::unique_ptr<iree_vm_instance_t, decltype(&iree_vm_instance_release)>;
+static InstancePtr MakeInstance() {
+ iree_vm_instance_t* instance = NULL;
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance));
+ return InstancePtr(instance, iree_vm_instance_release);
+}
+
class A : public iree::vm::RefObject<A> {
public:
static iree_vm_ref_type_t kTypeID;
@@ -43,54 +53,60 @@
};
template <typename T>
-static iree_vm_ref_t MakeRef(const char* type_name) {
+static iree_vm_ref_t MakeRef(InstancePtr& instance, const char* type_name) {
// Safe to do multiple times, so we do it to ensure the tests don't care what
// order they run in/don't need to preregister types.
static iree_vm_ref_type_descriptor_t descriptor = {0};
- if (descriptor.type == IREE_VM_REF_TYPE_NULL) {
- descriptor.type_name = iree_make_cstring_view(type_name);
- descriptor.offsetof_counter = T::offsetof_counter();
- descriptor.destroy = T::DirectDestroy;
- IREE_CHECK_OK(iree_vm_ref_register_type(&descriptor));
- T::kTypeID = descriptor.type;
- }
+ static iree_vm_ref_type_t registration = 0;
+ descriptor.type_name = iree_make_cstring_view(type_name);
+ descriptor.offsetof_counter = T::offsetof_counter();
+ descriptor.destroy = T::DirectDestroy;
+ IREE_CHECK_OK(iree_vm_instance_register_type(instance.get(), &descriptor,
+ ®istration));
+ T::kTypeID = registration;
iree_vm_ref_t ref = {0};
IREE_CHECK_OK(iree_vm_ref_wrap_assign(new T(), T::kTypeID, &ref));
return ref;
}
+// WARNING: this is an implementation detail and must never be relied on - it's
+// only here to test the expected behavior.
static int32_t ReadCounter(iree_vm_ref_t* ref) {
- return iree_atomic_load_int32(
- (iree_atomic_ref_count_t*)(((uintptr_t)ref->ptr) + ref->offsetof_counter),
- iree_memory_order_seq_cst);
+ return iree_atomic_load_int32((iree_atomic_ref_count_t*)ref->ptr +
+ (ref->type & IREE_VM_REF_TYPE_TAG_BIT_MASK),
+ iree_memory_order_seq_cst);
}
static iree_vm_ref_type_t kCTypeID = IREE_VM_REF_TYPE_NULL;
-static void RegisterTypeC() {
+static void RegisterTypeC(InstancePtr& instance) {
static iree_vm_ref_type_descriptor_t descriptor = {0};
- if (descriptor.type == IREE_VM_REF_TYPE_NULL) {
- descriptor.type_name = iree_make_cstring_view("CType");
- descriptor.offsetof_counter = offsetof(ref_object_c_t, ref_object.counter);
- descriptor.destroy =
- +[](void* ptr) { delete reinterpret_cast<ref_object_c_t*>(ptr); };
- IREE_CHECK_OK(iree_vm_ref_register_type(&descriptor));
- kCTypeID = descriptor.type;
- }
+ static iree_vm_ref_type_t registration = 0;
+ descriptor.type_name = iree_make_cstring_view("CType");
+ descriptor.offsetof_counter = offsetof(ref_object_c_t, ref_object.counter) /
+ IREE_VM_REF_COUNTER_ALIGNMENT;
+ descriptor.destroy =
+ +[](void* ptr) { delete reinterpret_cast<ref_object_c_t*>(ptr); };
+ IREE_CHECK_OK(iree_vm_instance_register_type(instance.get(), &descriptor,
+ ®istration));
+ kCTypeID = registration;
}
// Tests type registration and lookup.
TEST(VMRefTest, TypeRegistration) {
- RegisterTypeC();
- ASSERT_NE(nullptr, iree_vm_ref_lookup_registered_type(
- iree_make_cstring_view("CType")));
- ASSERT_EQ(nullptr, iree_vm_ref_lookup_registered_type(
- iree_make_cstring_view("asodjfaoisdjfaoisdfj")));
+ auto instance = MakeInstance();
+ RegisterTypeC(instance);
+ ASSERT_NE(0, iree_vm_instance_lookup_type(instance.get(),
+ iree_make_cstring_view("CType")));
+ ASSERT_EQ(
+ 0, iree_vm_instance_lookup_type(
+ instance.get(), iree_make_cstring_view("asodjfaoisdjfaoisdfj")));
}
// Tests wrapping a simple C struct.
TEST(VMRefTest, WrappingCStruct) {
- RegisterTypeC();
+ auto instance = MakeInstance();
+ RegisterTypeC(instance);
iree_vm_ref_t ref = {0};
IREE_EXPECT_OK(iree_vm_ref_wrap_assign(new ref_object_c_t(), kCTypeID, &ref));
EXPECT_EQ(1, ReadCounter(&ref));
@@ -110,17 +126,21 @@
int DoSomething() override { return 123 + allocated_derived_types; }
};
- static iree_vm_ref_type_descriptor_t descriptor;
+ auto instance = MakeInstance();
+
+ static iree_vm_ref_type_descriptor_t descriptor = {0};
+ static iree_vm_ref_type_t registration = 0;
descriptor.type_name = iree_make_cstring_view("BaseType");
descriptor.offsetof_counter = BaseType::offsetof_counter();
descriptor.destroy = BaseType::DirectDestroy;
- IREE_ASSERT_OK(iree_vm_ref_register_type(&descriptor));
+ IREE_ASSERT_OK(iree_vm_instance_register_type(instance.get(), &descriptor,
+ ®istration));
allocated_derived_types = 0;
iree_vm_ref_t ref = {0};
IREE_EXPECT_OK(
- iree_vm_ref_wrap_assign(new DerivedType(), descriptor.type, &ref));
+ iree_vm_ref_wrap_assign(new DerivedType(), registration, &ref));
EXPECT_EQ(1, ReadCounter(&ref));
EXPECT_EQ(1, allocated_derived_types);
@@ -130,19 +150,10 @@
EXPECT_EQ(0, allocated_derived_types);
}
-// Tests that wrapping a type that has not been registered fails.
-TEST(VMRefTest, WrappingRequriesTypeRegistration) {
- iree_vm_ref_t ref = {0};
- int dummy = 0;
- iree_status_t status = iree_vm_ref_wrap_assign(
- &dummy, static_cast<iree_vm_ref_type_t>(1234), &ref);
- IREE_EXPECT_STATUS_IS(IREE_STATUS_INVALID_ARGUMENT, status);
- iree_status_free(status);
-}
-
// Tests that wrapping releases any existing ref in out_ref.
TEST(VMRefTest, WrappingReleasesExisting) {
- RegisterTypeC();
+ auto instance = MakeInstance();
+ RegisterTypeC(instance);
iree_vm_ref_t ref = {0};
iree_vm_ref_wrap_assign(new ref_object_c_t(), kCTypeID, &ref);
EXPECT_EQ(1, ReadCounter(&ref));
@@ -161,7 +172,8 @@
// Tests type checks.
TEST(VMRefTest, Check) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
IREE_EXPECT_OK(iree_vm_ref_check(a_ref, A::kTypeID));
iree_status_t status = iree_vm_ref_check(a_ref, B::kTypeID);
IREE_EXPECT_STATUS_IS(IREE_STATUS_INVALID_ARGUMENT, status);
@@ -178,7 +190,8 @@
// Tests that retaining into itself is a no-op.
TEST(VMRefTest, RetainIntoSelf) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
EXPECT_EQ(1, ReadCounter(&a_ref));
iree_vm_ref_retain(&a_ref, &a_ref);
EXPECT_EQ(1, ReadCounter(&a_ref));
@@ -187,8 +200,9 @@
// Tests that retaining into out_ref releases the existing contents.
TEST(VMRefTest, RetainReleasesExisting) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
- iree_vm_ref_t b_ref = MakeRef<B>("BType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
+ iree_vm_ref_t b_ref = MakeRef<B>(instance, "BType");
iree_vm_ref_retain(&a_ref, &b_ref);
EXPECT_EQ(1, iree_vm_ref_equal(&a_ref, &b_ref));
EXPECT_EQ(2, ReadCounter(&a_ref));
@@ -206,7 +220,8 @@
// Tests that types are verified and retains fail if types don't match.
TEST(VMRefTest, RetainChecked) {
- iree_vm_ref_t a_ref_0 = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref_0 = MakeRef<A>(instance, "AType");
iree_vm_ref_t a_ref_1 = {0};
IREE_EXPECT_OK(iree_vm_ref_retain_checked(&a_ref_0, A::kTypeID, &a_ref_1));
iree_vm_ref_release(&a_ref_0);
@@ -223,7 +238,8 @@
// Tests that is_move=false increments the ref count.
TEST(VMRefTest, RetainOrMoveRetaining) {
- iree_vm_ref_t a_ref_0 = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref_0 = MakeRef<A>(instance, "AType");
iree_vm_ref_t a_ref_1 = {0};
iree_vm_ref_retain_or_move(/*is_move=*/0, &a_ref_0, &a_ref_1);
EXPECT_EQ(1, iree_vm_ref_equal(&a_ref_0, &a_ref_1));
@@ -234,7 +250,8 @@
// Tests that is_move=true does not increment the ref count.
TEST(VMRefTest, RetainOrMoveMoving) {
- iree_vm_ref_t a_ref_0 = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref_0 = MakeRef<A>(instance, "AType");
iree_vm_ref_t a_ref_1 = {0};
iree_vm_ref_retain_or_move(/*is_move=*/1, &a_ref_0, &a_ref_1);
IREE_EXPECT_OK(iree_vm_ref_check(a_ref_0, IREE_VM_REF_TYPE_NULL));
@@ -243,7 +260,8 @@
// Tests that retaining into itself just increments the ref count.
TEST(VMRefTest, RetainOrMoveRetainingIntoSelf) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
EXPECT_EQ(1, ReadCounter(&a_ref));
iree_vm_ref_retain_or_move(/*is_move=*/0, &a_ref, &a_ref);
EXPECT_EQ(1, ReadCounter(&a_ref));
@@ -252,7 +270,8 @@
// Tests that moving into itself is a no-op.
TEST(VMRefTest, RetainOrMoveMovingIntoSelf) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
iree_vm_ref_retain_or_move(/*is_move=*/1, &a_ref, &a_ref);
IREE_EXPECT_OK(iree_vm_ref_check(a_ref, A::kTypeID));
iree_vm_ref_release(&a_ref);
@@ -260,8 +279,9 @@
// Tests that retaining into out_ref releases the existing contents.
TEST(VMRefTest, RetainOrMoveRetainingReleasesExisting) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
- iree_vm_ref_t b_ref = MakeRef<B>("BType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
+ iree_vm_ref_t b_ref = MakeRef<B>(instance, "BType");
iree_vm_ref_retain_or_move(/*is_move=*/0, &a_ref, &b_ref);
EXPECT_EQ(1, iree_vm_ref_equal(&a_ref, &b_ref));
EXPECT_EQ(2, ReadCounter(&a_ref));
@@ -271,8 +291,9 @@
// Tests that moving into out_ref releases the existing contents.
TEST(VMRefTest, RetainOrMoveMovingReleasesExisting) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
- iree_vm_ref_t b_ref = MakeRef<B>("BType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
+ iree_vm_ref_t b_ref = MakeRef<B>(instance, "BType");
iree_vm_ref_retain_or_move(/*is_move=*/1, &a_ref, &b_ref);
EXPECT_EQ(0, iree_vm_ref_equal(&a_ref, &b_ref));
EXPECT_EQ(1, ReadCounter(&b_ref));
@@ -291,8 +312,10 @@
// Tests that retains/moves work when types match.
TEST(VMRefTest, RetainOrMoveCheckedMatch) {
+ auto instance = MakeInstance();
+
// Retain.
- iree_vm_ref_t a_ref_0 = MakeRef<A>("AType");
+ iree_vm_ref_t a_ref_0 = MakeRef<A>(instance, "AType");
iree_vm_ref_t a_ref_1 = {0};
IREE_EXPECT_OK(iree_vm_ref_retain_or_move_checked(
/*is_move=*/0, &a_ref_0, A::kTypeID, &a_ref_1));
@@ -302,7 +325,7 @@
iree_vm_ref_release(&a_ref_1);
// Move.
- iree_vm_ref_t b_ref_0 = MakeRef<B>("BType");
+ iree_vm_ref_t b_ref_0 = MakeRef<B>(instance, "BType");
iree_vm_ref_t b_ref_1 = {0};
IREE_EXPECT_OK(iree_vm_ref_retain_or_move_checked(
/*is_move=*/1, &b_ref_0, B::kTypeID, &b_ref_1));
@@ -313,8 +336,10 @@
// Tests that types are verified and retains/moves fail if types don't match.
TEST(VMRefTest, RetainOrMoveCheckedMismatch) {
+ auto instance = MakeInstance();
+
// Retain.
- iree_vm_ref_t a_ref_0 = MakeRef<A>("AType");
+ iree_vm_ref_t a_ref_0 = MakeRef<A>(instance, "AType");
iree_vm_ref_t a_ref_1 = {0};
iree_status_t status = iree_vm_ref_retain_or_move_checked(
/*is_move=*/0, &a_ref_0, B::kTypeID, &a_ref_1);
@@ -325,7 +350,7 @@
iree_vm_ref_release(&a_ref_0);
// Move.
- iree_vm_ref_t b_ref_0 = MakeRef<B>("BType");
+ iree_vm_ref_t b_ref_0 = MakeRef<B>(instance, "BType");
iree_vm_ref_t b_ref_1 = {0};
status = iree_vm_ref_retain_or_move_checked(
/*is_move=*/1, &b_ref_0, A::kTypeID, &b_ref_1);
@@ -337,16 +362,18 @@
// Tests that existing references are released when being overwritten.
TEST(VMRefTest, RetainOrMoveCheckedReleasesExistingNull) {
+ auto instance = MakeInstance();
iree_vm_ref_t null_ref = {0};
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
IREE_EXPECT_OK(iree_vm_ref_retain_or_move_checked(
/*is_move=*/0, &null_ref, A::kTypeID, &a_ref));
}
// Tests that existing references are released when being overwritten.
TEST(VMRefTest, RetainOrMoveCheckedReleasesExisting) {
- iree_vm_ref_t a_ref_0 = MakeRef<A>("AType");
- iree_vm_ref_t a_ref_1 = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref_0 = MakeRef<A>(instance, "AType");
+ iree_vm_ref_t a_ref_1 = MakeRef<A>(instance, "AType");
IREE_EXPECT_OK(iree_vm_ref_retain_or_move_checked(
/*is_move=*/1, &a_ref_0, A::kTypeID, &a_ref_1));
iree_vm_ref_release(&a_ref_1);
@@ -361,7 +388,8 @@
// Tests that assigning does not reset the source ref nor inc the ref count.
TEST(VMRefTest, Assign) {
- iree_vm_ref_t a_ref_0 = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref_0 = MakeRef<A>(instance, "AType");
iree_vm_ref_t a_ref_1 = {0};
iree_vm_ref_assign(&a_ref_0, &a_ref_1);
EXPECT_EQ(1, iree_vm_ref_equal(&a_ref_0, &a_ref_1));
@@ -371,7 +399,8 @@
// Tests that assigning into itself is a no-op.
TEST(VMRefTest, AssignSelf) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
iree_vm_ref_assign(&a_ref, &a_ref);
EXPECT_EQ(1, ReadCounter(&a_ref));
iree_vm_ref_release(&a_ref);
@@ -379,8 +408,9 @@
// Tests that assigning into out_ref releases the existing contents.
TEST(VMRefTest, AssignReleasesExisting) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
- iree_vm_ref_t b_ref = MakeRef<B>("BType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
+ iree_vm_ref_t b_ref = MakeRef<B>(instance, "BType");
iree_vm_ref_assign(&a_ref, &b_ref);
EXPECT_EQ(1, iree_vm_ref_equal(&a_ref, &b_ref));
EXPECT_EQ(1, ReadCounter(&a_ref));
@@ -397,7 +427,8 @@
// Tests that moving resets the source ref.
TEST(VMRefTest, MovingResetsSource) {
- iree_vm_ref_t a_ref_0 = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref_0 = MakeRef<A>(instance, "AType");
iree_vm_ref_t a_ref_1 = {0};
iree_vm_ref_move(&a_ref_0, &a_ref_1);
IREE_EXPECT_OK(iree_vm_ref_check(a_ref_0, IREE_VM_REF_TYPE_NULL));
@@ -406,7 +437,8 @@
// Tests that moving into itself is a no-op.
TEST(VMRefTest, MovingIntoSelf) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
iree_vm_ref_move(&a_ref, &a_ref);
IREE_EXPECT_OK(iree_vm_ref_check(a_ref, A::kTypeID));
iree_vm_ref_release(&a_ref);
@@ -414,8 +446,9 @@
// Tests that moving into out_ref releases the existing contents.
TEST(VMRefTest, MovingReleasesExisting) {
- iree_vm_ref_t a_ref_0 = MakeRef<A>("AType");
- iree_vm_ref_t a_ref_1 = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref_0 = MakeRef<A>(instance, "AType");
+ iree_vm_ref_t a_ref_1 = MakeRef<A>(instance, "AType");
iree_vm_ref_move(&a_ref_0, &a_ref_1);
iree_vm_ref_release(&a_ref_1);
}
@@ -431,7 +464,8 @@
// Tests comparing with self and against null.
TEST(VMRefTest, EqualitySelfOrNull) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
iree_vm_ref_t null_ref = {0};
EXPECT_EQ(1, iree_vm_ref_equal(&a_ref, &a_ref));
EXPECT_EQ(0, iree_vm_ref_equal(&a_ref, &null_ref));
@@ -441,8 +475,9 @@
// Tests comparing between different types.
TEST(VMRefTest, EqualityDifferentTypes) {
- iree_vm_ref_t a_ref = MakeRef<A>("AType");
- iree_vm_ref_t b_ref = MakeRef<B>("BType");
+ auto instance = MakeInstance();
+ iree_vm_ref_t a_ref = MakeRef<A>(instance, "AType");
+ iree_vm_ref_t b_ref = MakeRef<B>(instance, "BType");
EXPECT_EQ(0, iree_vm_ref_equal(&a_ref, &b_ref));
EXPECT_EQ(0, iree_vm_ref_equal(&b_ref, &a_ref));
iree_vm_ref_release(&b_ref);
@@ -450,3 +485,26 @@
}
} // namespace
+
+IREE_VM_DECLARE_TYPE_ADAPTERS(ref_object_c, ref_object_c_t);
+IREE_VM_DEFINE_TYPE_ADAPTERS(ref_object_c, ref_object_c_t);
+
+namespace {
+
+// Tests that C++ ref<T> instances can be initialized even if the type is not
+// yet registered. This happens if whatever C++ object/scope owns the instance
+// has fields that are initialized prior to the instance/module loading.
+TEST(VMRefTest, UnregisteredType) {
+ iree::vm::ref<ref_object_c_t> null_ref;
+ EXPECT_FALSE(null_ref);
+ EXPECT_EQ(nullptr, null_ref.get());
+ EXPECT_EQ(nullptr, null_ref.release());
+ EXPECT_EQ(0, null_ref.type());
+ null_ref.reset(); // don't die
+ auto retained_ref = iree::vm::retain_ref<ref_object_c_t>(nullptr);
+ EXPECT_FALSE(retained_ref);
+ auto assigned_ref = iree::vm::assign_ref<ref_object_c_t>(nullptr);
+ EXPECT_FALSE(assigned_ref);
+}
+
+} // namespace
diff --git a/runtime/src/iree/vm/test/emitc/module_test.cc b/runtime/src/iree/vm/test/emitc/module_test.cc
index 9d08bcc..bb85530 100644
--- a/runtime/src/iree/vm/test/emitc/module_test.cc
+++ b/runtime/src/iree/vm/test/emitc/module_test.cc
@@ -119,7 +119,8 @@
virtual void SetUp() {
const auto& test_params = GetParam();
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance_));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance_));
iree_vm_module_t* module_ = nullptr;
IREE_CHECK_OK(test_params.create_function(
diff --git a/runtime/src/iree/vm/type_def.h b/runtime/src/iree/vm/type_def.h
index 3de996c..dc648ca 100644
--- a/runtime/src/iree/vm/type_def.h
+++ b/runtime/src/iree/vm/type_def.h
@@ -23,40 +23,62 @@
// * i8: primitive value (value_type != 0)
// * !vm.ref<?>: any ref value (ref_type == IREE_VM_REF_TYPE_ANY)
// * !vm.ref<!foo>: ref value of type !foo (ref_type > 0)
+//
+// Implementation note: since type defs are used frequently and live in tables
+// and on the stack we pack the value and ref types together into a single
+// native machine word. This exploits the fact that iree_vm_ref_type_t is a
+// pointer to a struct that should always be aligned to some multiple of the
+// native machine word and we'll have low bits to spare.
typedef struct iree_vm_type_def_t {
- iree_vm_value_type_t value_type : 8;
- iree_vm_ref_type_t ref_type : 24;
+ uintptr_t value_type_bits : IREE_VM_REF_TYPE_TAG_BITS;
+ uintptr_t ref_type_bits : IREE_VM_REF_TYPE_PTR_BITS;
} iree_vm_type_def_t;
+static_assert(sizeof(iree_vm_type_def_t) == sizeof(uintptr_t),
+ "iree_vm_type_def_t should be a single native machine word");
+static_assert(
+ IREE_VM_VALUE_TYPE_MAX <= IREE_VM_REF_TYPE_TAG_BIT_MASK,
+ "iree_vm_value_type_t must fit within the iree_vm_ref_type_t tag bits");
-static inline iree_vm_type_def_t iree_vm_type_def_make_variant_type(void) {
- iree_vm_type_def_t result;
- result.value_type = IREE_VM_VALUE_TYPE_NONE;
- result.ref_type = IREE_VM_REF_TYPE_NULL;
- return result;
-}
-
-static inline iree_vm_type_def_t iree_vm_type_def_make_value_type(
- iree_vm_value_type_t value_type) {
- iree_vm_type_def_t result;
- result.value_type = value_type;
- result.ref_type = IREE_VM_REF_TYPE_NULL;
- return result;
-}
-
-static inline iree_vm_type_def_t iree_vm_type_def_make_ref_type(
- iree_vm_ref_type_t ref_type) {
- iree_vm_type_def_t result;
- result.value_type = IREE_VM_VALUE_TYPE_NONE;
- result.ref_type = ref_type;
- return result;
-}
+#define iree_vm_type_def_as_value(v) (iree_vm_value_type_t)((v).value_type_bits)
+#define iree_vm_type_def_as_ref(v) \
+ (((iree_vm_ref_type_t)(v).ref_type_bits) << IREE_VM_REF_TYPE_TAG_BITS)
#define iree_vm_type_def_is_value(v) \
- ((v)->value_type != IREE_VM_VALUE_TYPE_NONE)
-#define iree_vm_type_def_is_ref(v) ((v)->ref_type != IREE_VM_REF_TYPE_NULL)
-#define iree_vm_type_def_is_variant(v) \
- ((v)->value_type == IREE_VM_VALUE_TYPE_NONE && \
- (v)->ref_type == IREE_VM_REF_TYPE_NULL)
+ (iree_vm_type_def_as_value(v) != IREE_VM_VALUE_TYPE_NONE)
+#define iree_vm_type_def_is_ref(v) \
+ (iree_vm_type_def_as_ref(v) != IREE_VM_REF_TYPE_NULL)
+#define iree_vm_type_def_is_variant(v) \
+ (iree_vm_type_def_as_value(v) == IREE_VM_VALUE_TYPE_NONE && \
+ iree_vm_type_def_as_ref(v) == IREE_VM_REF_TYPE_NULL)
+#define iree_vm_type_def_is_undefined(v) iree_vm_type_def_is_variant(v)
+
+static bool iree_vm_type_def_equal(iree_vm_type_def_t a, iree_vm_type_def_t b) {
+ return a.value_type_bits == b.value_type_bits &&
+ a.ref_type_bits == b.ref_type_bits;
+}
+
+static inline iree_vm_type_def_t iree_vm_make_undefined_type_def(void) {
+ iree_vm_type_def_t result;
+ result.value_type_bits = IREE_VM_VALUE_TYPE_NONE;
+ result.ref_type_bits = IREE_VM_REF_TYPE_NULL;
+ return result;
+}
+
+static inline iree_vm_type_def_t iree_vm_make_value_type_def(
+ iree_vm_value_type_t value_type) {
+ iree_vm_type_def_t result;
+ result.value_type_bits = value_type;
+ result.ref_type_bits = IREE_VM_REF_TYPE_NULL;
+ return result;
+}
+
+static inline iree_vm_type_def_t iree_vm_make_ref_type_def(
+ iree_vm_ref_type_t ref_type) {
+ iree_vm_type_def_t result;
+ result.value_type_bits = IREE_VM_VALUE_TYPE_NONE;
+ result.ref_type_bits = ref_type >> IREE_VM_REF_TYPE_TAG_BITS;
+ return result;
+}
#ifdef __cplusplus
} // extern "C"
diff --git a/runtime/src/iree/vm/value.h b/runtime/src/iree/vm/value.h
index 445d80f..eacbf96 100644
--- a/runtime/src/iree/vm/value.h
+++ b/runtime/src/iree/vm/value.h
@@ -38,6 +38,9 @@
IREE_VM_VALUE_TYPE_MAX = IREE_VM_VALUE_TYPE_F64,
IREE_VM_VALUE_TYPE_COUNT = IREE_VM_VALUE_TYPE_MAX + 1, // used for lookup
} iree_vm_value_type_t;
+static_assert(
+ IREE_VM_VALUE_TYPE_MAX < (1 << IREE_VM_REF_TYPE_TAG_BITS),
+ "bitpacking in iree_vm_type_def_t requires this enum to be small");
// Maximum size, in bytes, of any value type we can represent.
#define IREE_VM_VALUE_STORAGE_SIZE 8
diff --git a/runtime/src/iree/vm/variant.h b/runtime/src/iree/vm/variant.h
index dfff9ed..8acf2c1 100644
--- a/runtime/src/iree/vm/variant.h
+++ b/runtime/src/iree/vm/variant.h
@@ -37,31 +37,31 @@
// Returns an empty variant.
static inline iree_vm_variant_t iree_vm_variant_empty(void) {
iree_vm_variant_t result;
- result.type = iree_vm_type_def_make_variant_type();
+ result.type = iree_vm_make_undefined_type_def();
result.ref = iree_vm_ref_null();
return result;
}
// Returns true if |variant| is empty (no value/NULL ref).
static inline bool iree_vm_variant_is_empty(iree_vm_variant_t variant) {
- return iree_vm_type_def_is_variant(&variant.type);
+ return iree_vm_type_def_is_variant(variant.type);
}
// Returns true if |variant| represents a primitive value.
static inline bool iree_vm_variant_is_value(iree_vm_variant_t variant) {
- return iree_vm_type_def_is_value(&variant.type);
+ return iree_vm_type_def_is_value(variant.type);
}
// Returns true if |variant| represents a non-NULL ref type.
static inline bool iree_vm_variant_is_ref(iree_vm_variant_t variant) {
- return iree_vm_type_def_is_ref(&variant.type);
+ return iree_vm_type_def_is_ref(variant.type);
}
// Makes a variant containing the given primitive |value|.
static inline iree_vm_variant_t iree_vm_make_variant_value(
iree_vm_value_t value) {
iree_vm_variant_t result = iree_vm_variant_empty();
- result.type.value_type = value.type;
+ result.type.value_type_bits = value.type;
memcpy(result.value_storage, value.value_storage,
sizeof(result.value_storage));
return result;
@@ -71,7 +71,7 @@
static inline iree_vm_variant_t iree_vm_make_variant_ref_assign(
iree_vm_ref_t ref) {
iree_vm_variant_t result = iree_vm_variant_empty();
- result.type.ref_type = ref.type;
+ result.type.ref_type_bits = ref.type >> IREE_VM_REF_TYPE_TAG_BITS;
result.ref = ref;
return result;
}
@@ -81,7 +81,7 @@
// iree_vm_value_make_none.
static inline iree_vm_value_t iree_vm_variant_value(iree_vm_variant_t variant) {
iree_vm_value_t value;
- value.type = variant.type.value_type;
+ value.type = (iree_vm_value_type_t)variant.type.value_type_bits;
memcpy(value.value_storage, variant.value_storage,
sizeof(value.value_storage));
return value;
diff --git a/samples/custom_module/async/main.c b/samples/custom_module/async/main.c
index 5e2a768..3ba492b 100644
--- a/samples/custom_module/async/main.c
+++ b/samples/custom_module/async/main.c
@@ -77,9 +77,11 @@
}
iree_vm_list_t* inputs = NULL;
- IREE_CHECK_OK(iree_vm_list_create(NULL, 1, host_allocator, &inputs));
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 1,
+ host_allocator, &inputs));
iree_vm_list_t* outputs = NULL;
- IREE_CHECK_OK(iree_vm_list_create(NULL, 1, host_allocator, &outputs));
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 1,
+ host_allocator, &outputs));
// Pass in the tensor<?xi32> arg:
const int32_t input_data[5] = {1, 2, 3, 4, 5};
diff --git a/samples/custom_module/basic/main.c b/samples/custom_module/basic/main.c
index 557a235..fe54540 100644
--- a/samples/custom_module/basic/main.c
+++ b/samples/custom_module/basic/main.c
@@ -45,11 +45,8 @@
// Create the root isolated VM instance that we can create contexts within.
iree_vm_instance_t* instance = NULL;
- IREE_CHECK_OK(iree_vm_instance_create(allocator, &instance));
-
- // Ensure custom types are registered before loading modules that use them.
- // This only needs to be done once.
- IREE_CHECK_OK(iree_custom_module_basic_register_types(instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ allocator, &instance));
// Create the custom module that can be reused across contexts.
iree_vm_module_t* custom_module = NULL;
diff --git a/samples/custom_module/basic/module.cc b/samples/custom_module/basic/module.cc
index 9f846d0..c46a7b4 100644
--- a/samples/custom_module/basic/module.cc
+++ b/samples/custom_module/basic/module.cc
@@ -65,17 +65,27 @@
iree_allocator_free(string->allocator, ptr);
}
-extern "C" iree_status_t iree_custom_module_basic_register_types(
+static iree_vm_ref_type_descriptor_t iree_custom_string_descriptor_storage = {
+ 0};
+
+// Registers types provided by the custom module.
+// We must call this before any of our types can be resolved.
+iree_status_t iree_custom_module_basic_register_types(
iree_vm_instance_t* instance) {
- if (iree_custom_string_descriptor.type) {
- return iree_ok_status(); // Already registered.
- }
- iree_custom_string_descriptor.type_name =
- iree_make_cstring_view("custom.string");
- iree_custom_string_descriptor.offsetof_counter =
- offsetof(iree_custom_string_t, ref_object.counter);
- iree_custom_string_descriptor.destroy = iree_custom_string_destroy;
- return iree_vm_ref_register_type(&iree_custom_string_descriptor);
+ iree_custom_string_descriptor_storage.destroy = iree_custom_string_destroy;
+ iree_custom_string_descriptor_storage.type_name = IREE_SV("custom.string");
+ iree_custom_string_descriptor_storage.offsetof_counter =
+ offsetof(iree_custom_string_t, ref_object.counter) /
+ IREE_VM_REF_COUNTER_ALIGNMENT;
+ return iree_vm_instance_register_type(instance,
+ &iree_custom_string_descriptor_storage,
+ &iree_custom_string_registration);
+}
+
+static void iree_custom_module_basic_unregister_types(
+ iree_vm_instance_t* instance) {
+ iree_vm_instance_unregister_type(instance,
+ &iree_custom_string_descriptor_storage);
}
//===----------------------------------------------------------------------===//
@@ -94,8 +104,8 @@
// it must synchronize itself.
class CustomModuleState final {
public:
- explicit CustomModuleState(iree_allocator_t allocator)
- : allocator_(allocator) {}
+ explicit CustomModuleState(iree_allocator_t host_allocator)
+ : host_allocator_(host_allocator) {}
~CustomModuleState() = default;
// Creates a new string with a copy of the given string data.
@@ -103,7 +113,8 @@
StatusOr<vm::ref<iree_custom_string_t>> StringCreate(
iree_string_view_t data) {
vm::ref<iree_custom_string_t> string;
- IREE_RETURN_IF_ERROR(iree_custom_string_create(data, allocator_, &string));
+ IREE_RETURN_IF_ERROR(
+ iree_custom_string_create(data, host_allocator_, &string));
fprintf(stdout, "CREATE %.*s\n", static_cast<int>(string->value.size),
string->value.data);
fflush(stdout);
@@ -140,7 +151,7 @@
private:
// Allocator that the caller requested we use for any allocations we need to
// perform during operation.
- iree_allocator_t allocator_ = iree_allocator_system();
+ iree_allocator_t host_allocator_;
};
// Function table mapping imported function names to their implementation.
@@ -169,6 +180,10 @@
public:
using vm::NativeModule<CustomModuleState>::NativeModule;
+ ~CustomModule() override {
+ iree_custom_module_basic_unregister_types(instance());
+ }
+
// Creates per-context state when the module is added to a new context.
// May be called from any thread.
StatusOr<std::unique_ptr<CustomModuleState>> CreateState(
@@ -187,10 +202,17 @@
iree_vm_module_t** out_module) {
IREE_ASSERT_ARGUMENT(out_module);
*out_module = NULL;
+
+ // Register the types used by the module.
+ // The CustomModule destructor will unregister them when it's done.
+ // Unregistration isn't strictly required in some cases but is good practice.
+ iree_custom_module_basic_register_types(instance);
+
auto module = std::make_unique<CustomModule>(
"custom", /*version=*/0, instance, allocator,
iree::span<const vm::NativeFunction<CustomModuleState>>(
kCustomModuleFunctions));
+
*out_module = module.release()->interface();
return iree_ok_status();
}
diff --git a/samples/custom_module/basic/module.h b/samples/custom_module/basic/module.h
index c786f6b..1031871 100644
--- a/samples/custom_module/basic/module.h
+++ b/samples/custom_module/basic/module.h
@@ -31,6 +31,8 @@
iree_custom_string_t** out_string);
// Registers types provided by the custom module.
+// Not required to be called unless trying to create types from the module
+// before creating the module (rare).
iree_status_t iree_custom_module_basic_register_types(
iree_vm_instance_t* instance);
diff --git a/samples/custom_module/sync/main.c b/samples/custom_module/sync/main.c
index 4095dfe..21a438b 100644
--- a/samples/custom_module/sync/main.c
+++ b/samples/custom_module/sync/main.c
@@ -80,9 +80,11 @@
fflush(stdout);
iree_vm_list_t* inputs = NULL;
- IREE_CHECK_OK(iree_vm_list_create(NULL, 1, host_allocator, &inputs));
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 1,
+ host_allocator, &inputs));
iree_vm_list_t* outputs = NULL;
- IREE_CHECK_OK(iree_vm_list_create(NULL, 1, host_allocator, &outputs));
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 1,
+ host_allocator, &outputs));
// Pass in the tensor<?xi32> arg:
const int32_t input_data[5] = {1, 2, 3, 4, 5};
diff --git a/samples/emitc_modules/add_module_test.cc b/samples/emitc_modules/add_module_test.cc
index de240b4..8a198ec 100644
--- a/samples/emitc_modules/add_module_test.cc
+++ b/samples/emitc_modules/add_module_test.cc
@@ -17,7 +17,8 @@
class VMAddModuleTest : public ::testing::Test {
protected:
virtual void SetUp() {
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance_));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance_));
iree_vm_module_t* add_module = nullptr;
IREE_CHECK_OK(
@@ -47,13 +48,15 @@
// Setup I/O lists and pass in the argument. The result list will be
// populated upon return.
vm::ref<iree_vm_list_t> input_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 1, iree_allocator_system(), &input_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 1, iree_allocator_system(),
+ &input_list));
auto arg_value = iree_vm_value_make_i32(arg);
IREE_RETURN_IF_ERROR(iree_vm_list_push_value(input_list.get(), &arg_value));
vm::ref<iree_vm_list_t> output_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 1, iree_allocator_system(), &output_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 1, iree_allocator_system(),
+ &output_list));
// Invoke the entry function to do our work. Runs synchronously.
IREE_RETURN_IF_ERROR(
@@ -80,8 +83,9 @@
// Setup I/O lists and pass in the argument. The result list will be
// populated upon return.
vm::ref<iree_vm_list_t> input_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 1, iree_allocator_system(), &input_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 1, iree_allocator_system(),
+ &input_list));
auto arg0_value = iree_vm_value_make_i32(arg0);
auto arg1_value = iree_vm_value_make_i32(arg1);
IREE_RETURN_IF_ERROR(
@@ -89,8 +93,9 @@
IREE_RETURN_IF_ERROR(
iree_vm_list_push_value(input_list.get(), &arg1_value));
vm::ref<iree_vm_list_t> output_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 1, iree_allocator_system(), &output_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 1, iree_allocator_system(),
+ &output_list));
// Invoke the entry function to do our work. Runs synchronously.
IREE_RETURN_IF_ERROR(
diff --git a/samples/emitc_modules/import_module_test.cc b/samples/emitc_modules/import_module_test.cc
index 7a78917..d7c3257 100644
--- a/samples/emitc_modules/import_module_test.cc
+++ b/samples/emitc_modules/import_module_test.cc
@@ -17,7 +17,8 @@
class VMImportModuleTest : public ::testing::Test {
protected:
virtual void SetUp() {
- IREE_CHECK_OK(iree_vm_instance_create(iree_allocator_system(), &instance_));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance_));
iree_vm_module_t* module_a = nullptr;
IREE_CHECK_OK(
@@ -53,13 +54,15 @@
// Setup I/O lists and pass in the argument. The result list will be
// populated upon return.
vm::ref<iree_vm_list_t> input_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 1, iree_allocator_system(), &input_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 1, iree_allocator_system(),
+ &input_list));
auto arg_value = iree_vm_value_make_i32(arg);
IREE_RETURN_IF_ERROR(iree_vm_list_push_value(input_list.get(), &arg_value));
vm::ref<iree_vm_list_t> output_list;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/nullptr, 1, iree_allocator_system(), &output_list));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 1, iree_allocator_system(),
+ &output_list));
// Invoke the entry function to do our work. Runs synchronously.
IREE_RETURN_IF_ERROR(
diff --git a/samples/simple_embedding/device_vmvx_sync.c b/samples/simple_embedding/device_vmvx_sync.c
index d997ad1..fa5981c 100644
--- a/samples/simple_embedding/device_vmvx_sync.c
+++ b/samples/simple_embedding/device_vmvx_sync.c
@@ -24,7 +24,8 @@
iree_hal_sync_device_params_initialize(¶ms);
iree_vm_instance_t* instance = NULL;
- IREE_RETURN_IF_ERROR(iree_vm_instance_create(host_allocator, &instance));
+ IREE_RETURN_IF_ERROR(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ host_allocator, &instance));
iree_hal_executable_loader_t* loader = NULL;
iree_status_t status = iree_hal_vmvx_module_loader_create(
diff --git a/samples/simple_embedding/simple_embedding.c b/samples/simple_embedding/simple_embedding.c
index 94272de..2d1dd94 100644
--- a/samples/simple_embedding/simple_embedding.c
+++ b/samples/simple_embedding/simple_embedding.c
@@ -32,8 +32,8 @@
iree_status_t Run() {
iree_vm_instance_t* instance = NULL;
- IREE_RETURN_IF_ERROR(
- iree_vm_instance_create(iree_allocator_system(), &instance));
+ IREE_RETURN_IF_ERROR(iree_vm_instance_create(
+ IREE_VM_TYPE_CAPACITY_DEFAULT, iree_allocator_system(), &instance));
IREE_RETURN_IF_ERROR(iree_hal_module_register_all_types(instance));
iree_hal_device_t* device = NULL;
@@ -97,10 +97,10 @@
// Setup call inputs with our buffers.
iree_vm_list_t* inputs = NULL;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/NULL,
- /*capacity=*/2, iree_allocator_system(), &inputs),
- "can't allocate input vm list");
+ IREE_RETURN_IF_ERROR(
+ iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ /*capacity=*/2, iree_allocator_system(), &inputs),
+ "can't allocate input vm list");
iree_vm_ref_t arg0_buffer_view_ref =
iree_hal_buffer_view_move_ref(arg0_buffer_view);
@@ -114,10 +114,10 @@
// Prepare outputs list to accept the results from the invocation.
// The output vm list is allocated statically.
iree_vm_list_t* outputs = NULL;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(
- /*element_type=*/NULL,
- /*capacity=*/1, iree_allocator_system(), &outputs),
- "can't allocate output vm list");
+ IREE_RETURN_IF_ERROR(
+ iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ /*capacity=*/1, iree_allocator_system(), &outputs),
+ "can't allocate output vm list");
// Synchronously invoke the function.
IREE_RETURN_IF_ERROR(iree_vm_invoke(
@@ -126,8 +126,7 @@
// Get the result buffers from the invocation.
iree_hal_buffer_view_t* ret_buffer_view =
- (iree_hal_buffer_view_t*)iree_vm_list_get_ref_deref(
- outputs, 0, &iree_hal_buffer_view_descriptor);
+ iree_vm_list_get_buffer_view_assign(outputs, 0);
if (ret_buffer_view == NULL) {
return iree_make_status(IREE_STATUS_NOT_FOUND,
"can't find return buffer view");
diff --git a/tools/android/run_module_app/src/main.cc b/tools/android/run_module_app/src/main.cc
index 739c3fe..4d9b804 100644
--- a/tools/android/run_module_app/src/main.cc
+++ b/tools/android/run_module_app/src/main.cc
@@ -92,7 +92,8 @@
Status RunModule(const IreeModuleInvocation& invocation) {
iree_vm_instance_t* instance = nullptr;
IREE_RETURN_IF_ERROR(
- iree_vm_instance_create(iree_allocator_system(), &instance),
+ iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ iree_allocator_system(), &instance),
"creating instance");
IREE_RETURN_IF_ERROR(iree_hal_module_register_all_types(instance),
"registering HAL types");
@@ -145,8 +146,9 @@
iree_allocator_system(), &inputs));
vm::ref<iree_vm_list_t> outputs;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(/*element_type=*/nullptr, 16,
- iree_allocator_system(), &outputs));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 16, iree_allocator_system(),
+ &outputs));
LOGI("Execute @%s", function_name.c_str());
IREE_RETURN_IF_ERROR(
diff --git a/tools/iree-benchmark-module-main.cc b/tools/iree-benchmark-module-main.cc
index f04f4d1..b242b64 100644
--- a/tools/iree-benchmark-module-main.cc
+++ b/tools/iree-benchmark-module-main.cc
@@ -167,7 +167,7 @@
IREE_TRACE_FRAME_MARK();
vm::ref<iree_vm_list_t> outputs;
- IREE_CHECK_OK(iree_vm_list_create(/*element_type=*/nullptr, 16,
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 16,
iree_allocator_system(), &outputs));
// Benchmarking loop.
@@ -289,7 +289,7 @@
// Setup empty outputs.
vm::ref<iree_vm_list_t> outputs;
- IREE_CHECK_OK(iree_vm_list_create(/*element_type=*/nullptr, 16,
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 16,
host_allocator, &outputs));
invocation_outputs.push_back(std::move(outputs));
}
@@ -362,13 +362,13 @@
IREE_TRACE_FRAME_MARK();
vm::ref<iree_vm_list_t> inputs;
- IREE_CHECK_OK(iree_vm_list_create(/*element_type=*/nullptr, 16,
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 16,
iree_allocator_system(), &inputs));
iree_vm_value_t batch_size = iree_vm_value_make_i32(FLAG_batch_size);
IREE_CHECK_OK(iree_vm_list_push_value(inputs.get(), &batch_size));
vm::ref<iree_vm_list_t> outputs;
- IREE_CHECK_OK(iree_vm_list_create(/*element_type=*/nullptr, 16,
+ IREE_CHECK_OK(iree_vm_list_create(iree_vm_make_undefined_type_def(), 16,
iree_allocator_system(), &outputs));
// Benchmarking loop.
diff --git a/tools/iree-benchmark-trace-main.c b/tools/iree-benchmark-trace-main.c
index e07aedf..7951249 100644
--- a/tools/iree-benchmark-trace-main.c
+++ b/tools/iree-benchmark-trace-main.c
@@ -277,7 +277,8 @@
// Setup shared instance used for each benchmark.
iree_vm_instance_t* instance = NULL;
- IREE_CHECK_OK(iree_vm_instance_create(host_allocator, &instance));
+ IREE_CHECK_OK(iree_vm_instance_create(IREE_VM_TYPE_CAPACITY_DEFAULT,
+ host_allocator, &instance));
// Parse all of stdin right away. The traces we run may source things from it
// and because we're running in a benchmark loop it'll quickly drain. To
diff --git a/tools/iree-e2e-matmul-test.c b/tools/iree-e2e-matmul-test.c
index 68892af..a0fe5b1 100644
--- a/tools/iree-e2e-matmul-test.c
+++ b/tools/iree-e2e-matmul-test.c
@@ -302,7 +302,7 @@
iree_vm_type_def_t elem_type = iree_vm_list_element_type(src);
iree_host_size_t size = iree_vm_list_size(src);
iree_allocator_t allocator = iree_hal_allocator_host_allocator(hal_allocator);
- IREE_RETURN_IF_ERROR(iree_vm_list_create(&elem_type, size, allocator, dst));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(elem_type, size, allocator, dst));
IREE_RETURN_IF_ERROR(iree_vm_list_resize(*dst, size));
for (iree_host_size_t i = 0; i < size; ++i) {
iree_hal_buffer_view_t* src_elem = NULL;
@@ -312,7 +312,7 @@
src_elem, &dst_elem));
iree_vm_ref_t dst_elem_ref = {0};
IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_assign(
- dst_elem, iree_hal_buffer_view_type_id(), &dst_elem_ref));
+ dst_elem, iree_hal_buffer_view_type(), &dst_elem_ref));
IREE_RETURN_IF_ERROR(iree_vm_list_set_ref_move(*dst, i, &dst_elem_ref));
}
return iree_ok_status();
@@ -987,7 +987,7 @@
iree_host_size_t size = iree_vm_list_size(src_list);
iree_allocator_t allocator = iree_hal_allocator_host_allocator(hal_allocator);
IREE_RETURN_IF_ERROR(
- iree_vm_list_create(&elem_type, size, allocator, dst_list));
+ iree_vm_list_create(elem_type, size, allocator, dst_list));
IREE_RETURN_IF_ERROR(iree_vm_list_resize(*dst_list, size));
for (iree_host_size_t i = 0; i < size; ++i) {
iree_hal_buffer_view_t* src = NULL;
@@ -997,7 +997,7 @@
device, hal_allocator, src, mask[i], &dst));
iree_vm_ref_t dst_ref = {0};
IREE_RETURN_IF_ERROR(
- iree_vm_ref_wrap_assign(dst, iree_hal_buffer_view_type_id(), &dst_ref));
+ iree_vm_ref_wrap_assign(dst, iree_hal_buffer_view_type(), &dst_ref));
IREE_RETURN_IF_ERROR(iree_vm_list_set_ref_move(*dst_list, i, &dst_ref));
}
return iree_ok_status();
@@ -1039,7 +1039,7 @@
// Invoke the function to produce the actual result.
iree_vm_list_t* device_outputs = NULL;
if (iree_status_is_ok(status)) {
- status = iree_vm_list_create(/*element_type=*/NULL,
+ status = iree_vm_list_create(iree_vm_make_undefined_type_def(),
/*initial_capacity=*/8, replay->host_allocator,
&device_outputs);
}
@@ -1356,8 +1356,8 @@
}
iree_vm_instance_t* instance = NULL;
- iree_status_t status =
- iree_vm_instance_create(iree_allocator_system(), &instance);
+ iree_status_t status = iree_vm_instance_create(
+ IREE_VM_TYPE_CAPACITY_DEFAULT, iree_allocator_system(), &instance);
if (iree_status_is_ok(status)) {
status = run_trace_files(argc - 1, argv + 1, instance);
}
diff --git a/tools/iree-run-mlir-main.cc b/tools/iree-run-mlir-main.cc
index 0cc309c..bb1b4b5 100644
--- a/tools/iree-run-mlir-main.cc
+++ b/tools/iree-run-mlir-main.cc
@@ -371,8 +371,8 @@
// Prepare outputs list to accept the results from the invocation.
vm::ref<iree_vm_list_t> outputs;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(/*element_type=*/nullptr, 16,
- host_allocator, &outputs));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 16, host_allocator, &outputs));
// Synchronously invoke the function.
IREE_RETURN_IF_ERROR(iree_vm_invoke(
diff --git a/tools/iree-run-module-main.cc b/tools/iree-run-module-main.cc
index 7f5a439..aaf3837 100644
--- a/tools/iree-run-module-main.cc
+++ b/tools/iree-run-module-main.cc
@@ -136,8 +136,8 @@
&finish_fence));
vm::ref<iree_vm_list_t> outputs;
- IREE_RETURN_IF_ERROR(iree_vm_list_create(/*element_type=*/nullptr, 16,
- host_allocator, &outputs));
+ IREE_RETURN_IF_ERROR(iree_vm_list_create(iree_vm_make_undefined_type_def(),
+ 16, host_allocator, &outputs));
printf("EXEC @%s\n", function_name.c_str());
IREE_RETURN_IF_ERROR(
diff --git a/tools/iree-run-trace-main.c b/tools/iree-run-trace-main.c
index 502c7a2..fc9563b 100644
--- a/tools/iree-run-trace-main.c
+++ b/tools/iree-run-trace-main.c
@@ -363,8 +363,8 @@
}
iree_vm_instance_t* instance = NULL;
- iree_status_t status =
- iree_vm_instance_create(iree_allocator_system(), &instance);
+ iree_status_t status = iree_vm_instance_create(
+ IREE_VM_TYPE_CAPACITY_DEFAULT, iree_allocator_system(), &instance);
if (iree_status_is_ok(status)) {
status = iree_run_trace_files(argc - 1, argv + 1, instance);
}
