| // Copyright 2019 The IREE Authors | |
| // | |
| // Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| #include "iree/vm/bytecode_module.h" | |
| #include <stdbool.h> | |
| #include <stdint.h> | |
| #include <string.h> | |
| #include "iree/base/api.h" | |
| #include "iree/base/tracing.h" | |
| #include "iree/vm/api.h" | |
| #include "iree/vm/bytecode_module_impl.h" | |
| // Perform an strcmp between a flatbuffers string and an IREE string view. | |
| static bool iree_vm_flatbuffer_strcmp(flatbuffers_string_t lhs, | |
| iree_string_view_t rhs) { | |
| size_t lhs_size = flatbuffers_string_len(lhs); | |
| int x = strncmp(lhs, rhs.data, lhs_size < rhs.size ? lhs_size : rhs.size); | |
| return x != 0 ? x : lhs_size < rhs.size ? -1 : lhs_size > rhs.size; | |
| } | |
| // Resolves a type through either builtin rules or the ref registered types. | |
| static bool iree_vm_bytecode_module_resolve_type( | |
| iree_vm_TypeDef_table_t type_def, iree_vm_type_def_t* out_type) { | |
| memset(out_type, 0, sizeof(*out_type)); | |
| flatbuffers_string_t full_name = iree_vm_TypeDef_full_name(type_def); | |
| if (!flatbuffers_string_len(full_name)) { | |
| return false; | |
| } else if (iree_vm_flatbuffer_strcmp(full_name, | |
| iree_make_cstring_view("i8")) == 0) { | |
| out_type->value_type = IREE_VM_VALUE_TYPE_I8; | |
| return true; | |
| } else if (iree_vm_flatbuffer_strcmp(full_name, | |
| iree_make_cstring_view("i16")) == 0) { | |
| out_type->value_type = IREE_VM_VALUE_TYPE_I16; | |
| return true; | |
| } else if (iree_vm_flatbuffer_strcmp(full_name, | |
| iree_make_cstring_view("i32")) == 0) { | |
| out_type->value_type = IREE_VM_VALUE_TYPE_I32; | |
| return true; | |
| } else if (iree_vm_flatbuffer_strcmp(full_name, | |
| iree_make_cstring_view("i64")) == 0) { | |
| out_type->value_type = IREE_VM_VALUE_TYPE_I64; | |
| return true; | |
| } else if (iree_vm_flatbuffer_strcmp(full_name, | |
| iree_make_cstring_view("f32")) == 0) { | |
| out_type->value_type = IREE_VM_VALUE_TYPE_F32; | |
| return true; | |
| } else if (iree_vm_flatbuffer_strcmp(full_name, | |
| iree_make_cstring_view("f64")) == 0) { | |
| out_type->value_type = IREE_VM_VALUE_TYPE_F64; | |
| return true; | |
| } else if (iree_vm_flatbuffer_strcmp( | |
| full_name, iree_make_cstring_view("!vm.opaque")) == 0) { | |
| out_type->value_type = IREE_VM_VALUE_TYPE_NONE; | |
| out_type->ref_type = IREE_VM_REF_TYPE_NULL; | |
| return true; | |
| } else if (full_name[0] == '!') { | |
| // Note that we drop the ! prefix: | |
| iree_string_view_t type_name = {full_name + 1, | |
| flatbuffers_string_len(full_name) - 1}; | |
| if (iree_string_view_starts_with(type_name, | |
| iree_make_cstring_view("vm.list"))) { | |
| // This is a !vm.list<...> type. We don't actually care about the type as | |
| // we allow list types to be widened. Rewrite to just vm.list as that's | |
| // all we have registered. | |
| type_name = iree_make_cstring_view("vm.list"); | |
| } | |
| const iree_vm_ref_type_descriptor_t* type_descriptor = | |
| iree_vm_ref_lookup_registered_type(type_name); | |
| if (type_descriptor) { | |
| out_type->ref_type = type_descriptor->type; | |
| } | |
| return true; | |
| } | |
| return false; | |
| } | |
| // Resolves all types through either builtin rules or the ref registered types. | |
| // |type_table| can be omitted to just perform verification that all types are | |
| // registered. | |
| static iree_status_t iree_vm_bytecode_module_resolve_types( | |
| iree_vm_TypeDef_vec_t type_defs, iree_vm_type_def_t* type_table) { | |
| IREE_TRACE_ZONE_BEGIN(z0); | |
| iree_status_t status = iree_ok_status(); | |
| for (size_t i = 0; i < iree_vm_TypeDef_vec_len(type_defs); ++i) { | |
| iree_vm_TypeDef_table_t type_def = iree_vm_TypeDef_vec_at(type_defs, i); | |
| if (!iree_vm_bytecode_module_resolve_type(type_def, &type_table[i])) { | |
| status = iree_make_status(IREE_STATUS_NOT_FOUND, | |
| "no type registered with name '%s'", | |
| iree_vm_TypeDef_full_name(type_def)); | |
| break; | |
| } | |
| } | |
| IREE_TRACE_ZONE_END(z0); | |
| return status; | |
| } | |
| // Computes the total length of the flatbuffer and the base offset for any | |
| // concatenated rodata. | |
| static iree_host_size_t iree_vm_bytecode_module_flatbuffer_rodata_offset( | |
| iree_const_byte_span_t flatbuffer_data) { | |
| if (flatbuffer_data.data_length < sizeof(flatbuffers_uoffset_t)) return 0; | |
| size_t external_rodata_offset = 0; | |
| flatbuffers_read_size_prefix((void*)flatbuffer_data.data, | |
| &external_rodata_offset); | |
| external_rodata_offset += sizeof(flatbuffers_uoffset_t); | |
| return iree_host_align(external_rodata_offset, 128); | |
| } | |
| // Verifies the structure of the flatbuffer so that we can avoid doing so during | |
| // runtime. There are still some conditions we must be aware of (such as omitted | |
| // names on functions with internal linkage), however we shouldn't need to | |
| // bounds check anything within the flatbuffer after this succeeds. | |
| static iree_status_t iree_vm_bytecode_module_flatbuffer_verify( | |
| iree_const_byte_span_t flatbuffer_data) { | |
| if (!flatbuffer_data.data || flatbuffer_data.data_length < 16) { | |
| return iree_make_status( | |
| IREE_STATUS_INVALID_ARGUMENT, | |
| "flatbuffer data is not present or less than 16 bytes (%zu total)", | |
| flatbuffer_data.data_length); | |
| } | |
| // Run flatcc generated verification. This ensures all pointers are in-bounds | |
| // and that we can safely walk the file, but not that the actual contents of | |
| // the flatbuffer meet our expectations. | |
| int verify_ret = iree_vm_BytecodeModuleDef_verify_as_root( | |
| flatbuffer_data.data, flatbuffer_data.data_length); | |
| if (verify_ret != flatcc_verify_ok) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "flatbuffer verification failed: %s", | |
| flatcc_verify_error_string(verify_ret)); | |
| } | |
| const iree_host_size_t external_rodata_offset = | |
| iree_vm_bytecode_module_flatbuffer_rodata_offset(flatbuffer_data); | |
| iree_vm_BytecodeModuleDef_table_t module_def = | |
| iree_vm_BytecodeModuleDef_as_root(flatbuffer_data.data); | |
| flatbuffers_string_t name = iree_vm_BytecodeModuleDef_name(module_def); | |
| if (!flatbuffers_string_len(name)) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "module missing name field"); | |
| } | |
| iree_vm_TypeDef_vec_t types = iree_vm_BytecodeModuleDef_types(module_def); | |
| for (size_t i = 0; i < iree_vm_TypeDef_vec_len(types); ++i) { | |
| iree_vm_TypeDef_table_t type_def = iree_vm_TypeDef_vec_at(types, i); | |
| if (!type_def) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "types[%zu] missing body", i); | |
| } | |
| flatbuffers_string_t full_name = iree_vm_TypeDef_full_name(type_def); | |
| if (flatbuffers_string_len(full_name) <= 0) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "types[%zu] missing name", i); | |
| } | |
| } | |
| iree_vm_RodataSegmentDef_vec_t rodata_segments = | |
| iree_vm_BytecodeModuleDef_rodata_segments(module_def); | |
| for (size_t i = 0; i < iree_vm_RodataSegmentDef_vec_len(rodata_segments); | |
| ++i) { | |
| iree_vm_RodataSegmentDef_table_t segment = | |
| iree_vm_RodataSegmentDef_vec_at(rodata_segments, i); | |
| if (iree_vm_RodataSegmentDef_embedded_data_is_present(segment)) { | |
| continue; // embedded data is verified by FlatBuffers | |
| } | |
| uint64_t rodata_offset = | |
| iree_vm_RodataSegmentDef_external_data_offset(segment); | |
| uint64_t rodata_length = | |
| iree_vm_RodataSegmentDef_external_data_length(segment); | |
| uint64_t rodata_end = | |
| external_rodata_offset + rodata_offset + rodata_length; | |
| if (rodata_end >= flatbuffer_data.data_length) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "rodata[%zu] external reference out of range", i); | |
| } | |
| } | |
| iree_vm_ImportFunctionDef_vec_t imported_functions = | |
| iree_vm_BytecodeModuleDef_imported_functions(module_def); | |
| iree_vm_ExportFunctionDef_vec_t exported_functions = | |
| iree_vm_BytecodeModuleDef_exported_functions(module_def); | |
| iree_vm_FunctionDescriptor_vec_t function_descriptors = | |
| iree_vm_BytecodeModuleDef_function_descriptors(module_def); | |
| for (size_t i = 0; i < iree_vm_ImportFunctionDef_vec_len(imported_functions); | |
| ++i) { | |
| iree_vm_ImportFunctionDef_table_t import_def = | |
| iree_vm_ImportFunctionDef_vec_at(imported_functions, i); | |
| if (!import_def) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "imports[%zu] missing body", i); | |
| } | |
| flatbuffers_string_t full_name = | |
| iree_vm_ImportFunctionDef_full_name(import_def); | |
| if (!flatbuffers_string_len(full_name)) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "imports[%zu] missing full_name", i); | |
| } | |
| } | |
| for (size_t i = 0; i < iree_vm_ExportFunctionDef_vec_len(exported_functions); | |
| ++i) { | |
| iree_vm_ExportFunctionDef_table_t export_def = | |
| iree_vm_ExportFunctionDef_vec_at(exported_functions, i); | |
| if (!export_def) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "exports[%zu] missing body", i); | |
| } | |
| flatbuffers_string_t local_name = | |
| iree_vm_ExportFunctionDef_local_name(export_def); | |
| if (!flatbuffers_string_len(local_name)) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "exports[%zu] missing local_name", i); | |
| } | |
| iree_host_size_t internal_ordinal = | |
| iree_vm_ExportFunctionDef_internal_ordinal(export_def); | |
| if (internal_ordinal >= | |
| iree_vm_FunctionDescriptor_vec_len(function_descriptors)) { | |
| return iree_make_status( | |
| IREE_STATUS_INVALID_ARGUMENT, | |
| "exports[%zu] internal_ordinal out of bounds (0 < %zu < %zu)", i, | |
| internal_ordinal, | |
| iree_vm_FunctionDescriptor_vec_len(function_descriptors)); | |
| } | |
| } | |
| // Verify that we can properly handle the bytecode embedded in the module. | |
| // We require that major versions match and allow loading of older minor | |
| // versions (we keep changes backwards-compatible). | |
| const uint32_t bytecode_version = | |
| iree_vm_BytecodeModuleDef_bytecode_version(module_def); | |
| const uint32_t bytecode_version_major = bytecode_version >> 16; | |
| const uint32_t bytecode_version_minor = bytecode_version & 0xFFFF; | |
| if ((bytecode_version_major != IREE_VM_BYTECODE_VERSION_MAJOR) || | |
| (bytecode_version_minor > IREE_VM_BYTECODE_VERSION_MINOR)) { | |
| return iree_make_status( | |
| IREE_STATUS_INVALID_ARGUMENT, | |
| "bytecode version mismatch; runtime supports %d.%d, module has %d.%d", | |
| IREE_VM_BYTECODE_VERSION_MAJOR, IREE_VM_BYTECODE_VERSION_MINOR, | |
| bytecode_version_major, bytecode_version_minor); | |
| } | |
| flatbuffers_uint8_vec_t bytecode_data = | |
| iree_vm_BytecodeModuleDef_bytecode_data(module_def); | |
| for (size_t i = 0; | |
| i < iree_vm_FunctionDescriptor_vec_len(function_descriptors); ++i) { | |
| iree_vm_FunctionDescriptor_struct_t function_descriptor = | |
| iree_vm_FunctionDescriptor_vec_at(function_descriptors, i); | |
| if (function_descriptor->bytecode_offset < 0 || | |
| function_descriptor->bytecode_offset + | |
| function_descriptor->bytecode_length > | |
| flatbuffers_uint8_vec_len(bytecode_data)) { | |
| return iree_make_status( | |
| IREE_STATUS_INVALID_ARGUMENT, | |
| "functions[%zu] descriptor bytecode span out of range (0 < %d < %zu)", | |
| i, function_descriptor->bytecode_offset, | |
| flatbuffers_uint8_vec_len(bytecode_data)); | |
| } | |
| if (function_descriptor->i32_register_count > IREE_I32_REGISTER_COUNT || | |
| function_descriptor->ref_register_count > IREE_REF_REGISTER_COUNT) { | |
| return iree_make_status( | |
| IREE_STATUS_INVALID_ARGUMENT, | |
| "functions[%zu] descriptor register count out of range", i); | |
| } | |
| // TODO(benvanik): run bytecode verifier on contents. | |
| } | |
| return iree_ok_status(); | |
| } | |
| static iree_status_t iree_vm_bytecode_map_internal_ordinal( | |
| iree_vm_bytecode_module_t* module, iree_vm_function_t function, | |
| uint16_t* out_ordinal, | |
| iree_vm_FunctionSignatureDef_table_t* out_signature_def) { | |
| *out_ordinal = 0; | |
| if (out_signature_def) *out_signature_def = NULL; | |
| uint16_t ordinal = function.ordinal; | |
| iree_vm_FunctionSignatureDef_table_t signature_def = NULL; | |
| if (function.linkage == IREE_VM_FUNCTION_LINKAGE_EXPORT) { | |
| // Look up the internal ordinal index of this export in the function table. | |
| iree_vm_ExportFunctionDef_vec_t exported_functions = | |
| iree_vm_BytecodeModuleDef_exported_functions(module->def); | |
| IREE_ASSERT_LT(ordinal, | |
| iree_vm_ExportFunctionDef_vec_len(exported_functions), | |
| "export ordinal out of range (0 < %zu < %zu)", ordinal, | |
| iree_vm_ExportFunctionDef_vec_len(exported_functions)); | |
| iree_vm_ExportFunctionDef_table_t function_def = | |
| iree_vm_ExportFunctionDef_vec_at(exported_functions, function.ordinal); | |
| ordinal = iree_vm_ExportFunctionDef_internal_ordinal(function_def); | |
| signature_def = iree_vm_ExportFunctionDef_signature(function_def); | |
| } else { | |
| // TODO(benvanik): support querying the internal functions, which could be | |
| // useful for debugging. Or maybe we just drop them forever? | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "cannot map imported/internal functions; no entry " | |
| "in the function table"); | |
| } | |
| if (ordinal >= module->function_descriptor_count) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "function ordinal out of range (0 < %u < %zu)", | |
| function.ordinal, | |
| module->function_descriptor_count); | |
| } | |
| *out_ordinal = ordinal; | |
| if (out_signature_def) *out_signature_def = signature_def; | |
| return iree_ok_status(); | |
| } | |
| static void iree_vm_bytecode_module_destroy(void* self) { | |
| iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)self; | |
| IREE_TRACE_ZONE_BEGIN(z0); | |
| iree_allocator_free(module->flatbuffer_allocator, | |
| (void*)module->flatbuffer_data.data); | |
| module->flatbuffer_data = iree_make_const_byte_span(NULL, 0); | |
| module->flatbuffer_allocator = iree_allocator_null(); | |
| iree_allocator_free(module->allocator, module); | |
| IREE_TRACE_ZONE_END(z0); | |
| } | |
| static iree_string_view_t iree_vm_bytecode_module_name(void* self) { | |
| iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)self; | |
| flatbuffers_string_t name = iree_vm_BytecodeModuleDef_name(module->def); | |
| return iree_make_string_view(name, flatbuffers_string_len(name)); | |
| } | |
| static iree_vm_module_signature_t iree_vm_bytecode_module_signature( | |
| void* self) { | |
| iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)self; | |
| iree_vm_module_signature_t signature; | |
| memset(&signature, 0, sizeof(signature)); | |
| signature.import_function_count = iree_vm_ImportFunctionDef_vec_len( | |
| iree_vm_BytecodeModuleDef_imported_functions(module->def)); | |
| signature.export_function_count = iree_vm_ExportFunctionDef_vec_len( | |
| iree_vm_BytecodeModuleDef_exported_functions(module->def)); | |
| signature.internal_function_count = module->function_descriptor_count; | |
| return signature; | |
| } | |
| static iree_status_t iree_vm_bytecode_module_get_function( | |
| void* self, iree_vm_function_linkage_t linkage, iree_host_size_t ordinal, | |
| iree_vm_function_t* out_function, iree_string_view_t* out_name, | |
| iree_vm_function_signature_t* out_signature) { | |
| if (out_function) { | |
| memset(out_function, 0, sizeof(*out_function)); | |
| } | |
| if (out_name) { | |
| memset(out_name, 0, sizeof(*out_name)); | |
| } | |
| if (out_signature) { | |
| memset(out_signature, 0, sizeof(*out_signature)); | |
| } | |
| iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)self; | |
| flatbuffers_string_t name = NULL; | |
| iree_vm_FunctionSignatureDef_table_t signature = NULL; | |
| if (linkage == IREE_VM_FUNCTION_LINKAGE_IMPORT || | |
| linkage == IREE_VM_FUNCTION_LINKAGE_IMPORT_OPTIONAL) { | |
| iree_vm_ImportFunctionDef_vec_t imported_functions = | |
| iree_vm_BytecodeModuleDef_imported_functions(module->def); | |
| if (ordinal >= iree_vm_ImportFunctionDef_vec_len(imported_functions)) { | |
| return iree_make_status( | |
| IREE_STATUS_INVALID_ARGUMENT, | |
| "import ordinal out of range (0 < %zu < %zu)", ordinal, | |
| iree_vm_ImportFunctionDef_vec_len(imported_functions)); | |
| } | |
| iree_vm_ImportFunctionDef_table_t import_def = | |
| iree_vm_ImportFunctionDef_vec_at(imported_functions, ordinal); | |
| name = iree_vm_ImportFunctionDef_full_name(import_def); | |
| signature = iree_vm_ImportFunctionDef_signature(import_def); | |
| if (iree_all_bits_set(iree_vm_ImportFunctionDef_flags(import_def), | |
| iree_vm_ImportFlagBits_OPTIONAL)) { | |
| linkage = IREE_VM_FUNCTION_LINKAGE_IMPORT_OPTIONAL; | |
| } | |
| } else if (linkage == IREE_VM_FUNCTION_LINKAGE_EXPORT) { | |
| iree_vm_ExportFunctionDef_vec_t exported_functions = | |
| iree_vm_BytecodeModuleDef_exported_functions(module->def); | |
| if (ordinal >= iree_vm_ExportFunctionDef_vec_len(exported_functions)) { | |
| return iree_make_status( | |
| IREE_STATUS_INVALID_ARGUMENT, | |
| "export ordinal out of range (0 < %zu < %zu)", ordinal, | |
| iree_vm_ExportFunctionDef_vec_len(exported_functions)); | |
| } | |
| iree_vm_ExportFunctionDef_table_t export_def = | |
| iree_vm_ExportFunctionDef_vec_at(exported_functions, ordinal); | |
| name = iree_vm_ExportFunctionDef_local_name(export_def); | |
| signature = iree_vm_ExportFunctionDef_signature(export_def); | |
| } | |
| if (out_function) { | |
| out_function->module = &module->interface; | |
| out_function->linkage = linkage; | |
| out_function->ordinal = (uint16_t)ordinal; | |
| } | |
| if (out_name && name) { | |
| out_name->data = name; | |
| out_name->size = flatbuffers_string_len(name); | |
| } | |
| if (out_signature && signature) { | |
| flatbuffers_string_t calling_convention = | |
| iree_vm_FunctionSignatureDef_calling_convention(signature); | |
| out_signature->calling_convention.data = calling_convention; | |
| out_signature->calling_convention.size = | |
| flatbuffers_string_len(calling_convention); | |
| } | |
| return iree_ok_status(); | |
| } | |
| static iree_status_t iree_vm_bytecode_module_get_function_reflection_attr( | |
| void* self, iree_vm_function_linkage_t linkage, iree_host_size_t ordinal, | |
| iree_host_size_t index, iree_string_view_t* key, | |
| iree_string_view_t* value) { | |
| if (linkage != IREE_VM_FUNCTION_LINKAGE_EXPORT) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "only exported functions can be queried"); | |
| } | |
| iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)self; | |
| iree_vm_ExportFunctionDef_vec_t exported_functions = | |
| iree_vm_BytecodeModuleDef_exported_functions(module->def); | |
| if (ordinal >= iree_vm_ExportFunctionDef_vec_len(exported_functions)) { | |
| return iree_make_status( | |
| IREE_STATUS_INVALID_ARGUMENT, | |
| "function ordinal out of range (0 < %zu < %zu)", ordinal, | |
| iree_vm_ExportFunctionDef_vec_len(exported_functions)); | |
| } | |
| iree_vm_ExportFunctionDef_table_t function_def = | |
| iree_vm_ExportFunctionDef_vec_at(exported_functions, ordinal); | |
| iree_vm_FunctionSignatureDef_table_t signature_def = | |
| iree_vm_ExportFunctionDef_signature(function_def); | |
| if (!signature_def) { | |
| return iree_make_status( | |
| IREE_STATUS_NOT_FOUND, | |
| "reflection attribute at index %zu not found; no signature", index); | |
| } | |
| iree_vm_ReflectionAttrDef_vec_t reflection_attrs = | |
| iree_vm_FunctionSignatureDef_reflection_attrs(signature_def); | |
| if (!reflection_attrs || | |
| index >= iree_vm_ReflectionAttrDef_vec_len(reflection_attrs)) { | |
| return iree_make_status(IREE_STATUS_NOT_FOUND, | |
| "reflection attribute at index %zu not found", | |
| index); | |
| } | |
| iree_vm_ReflectionAttrDef_table_t attr = | |
| iree_vm_ReflectionAttrDef_vec_at(reflection_attrs, index); | |
| flatbuffers_string_t attr_key = iree_vm_ReflectionAttrDef_key(attr); | |
| flatbuffers_string_t attr_value = iree_vm_ReflectionAttrDef_value(attr); | |
| if (!flatbuffers_string_len(attr_key) || | |
| !flatbuffers_string_len(attr_value)) { | |
| // Because reflection metadata should not impose any overhead for the | |
| // non reflection case, we do not eagerly validate it on load -- instead | |
| // verify it structurally as needed. | |
| return iree_make_status(IREE_STATUS_FAILED_PRECONDITION, | |
| "reflection attribute missing fields"); | |
| } | |
| key->data = attr_key; | |
| key->size = flatbuffers_string_len(attr_key); | |
| value->data = attr_value; | |
| value->size = flatbuffers_string_len(attr_value); | |
| return iree_ok_status(); | |
| } | |
| static iree_status_t iree_vm_bytecode_module_lookup_function( | |
| void* self, iree_vm_function_linkage_t linkage, iree_string_view_t name, | |
| iree_vm_function_t* out_function) { | |
| IREE_ASSERT_ARGUMENT(out_function); | |
| memset(out_function, 0, sizeof(iree_vm_function_t)); | |
| if (iree_string_view_is_empty(name)) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "function name required for query"); | |
| } | |
| iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)self; | |
| out_function->linkage = linkage; | |
| out_function->module = &module->interface; | |
| // NOTE: we could organize exports alphabetically so we could bsearch. | |
| if (linkage == IREE_VM_FUNCTION_LINKAGE_IMPORT || | |
| linkage == IREE_VM_FUNCTION_LINKAGE_IMPORT_OPTIONAL) { | |
| iree_vm_ImportFunctionDef_vec_t imported_functions = | |
| iree_vm_BytecodeModuleDef_imported_functions(module->def); | |
| for (iree_host_size_t ordinal = 0; | |
| ordinal < iree_vm_ImportFunctionDef_vec_len(imported_functions); | |
| ++ordinal) { | |
| iree_vm_ImportFunctionDef_table_t import_def = | |
| iree_vm_ImportFunctionDef_vec_at(imported_functions, ordinal); | |
| if (iree_vm_flatbuffer_strcmp( | |
| iree_vm_ImportFunctionDef_full_name(import_def), name) == 0) { | |
| out_function->ordinal = ordinal; | |
| if (iree_all_bits_set(iree_vm_ImportFunctionDef_flags(import_def), | |
| iree_vm_ImportFlagBits_OPTIONAL)) { | |
| out_function->linkage = IREE_VM_FUNCTION_LINKAGE_IMPORT_OPTIONAL; | |
| } | |
| return iree_ok_status(); | |
| } | |
| } | |
| } else if (linkage == IREE_VM_FUNCTION_LINKAGE_EXPORT) { | |
| iree_vm_ExportFunctionDef_vec_t exported_functions = | |
| iree_vm_BytecodeModuleDef_exported_functions(module->def); | |
| for (iree_host_size_t ordinal = 0; | |
| ordinal < iree_vm_ExportFunctionDef_vec_len(exported_functions); | |
| ++ordinal) { | |
| iree_vm_ExportFunctionDef_table_t export_def = | |
| iree_vm_ExportFunctionDef_vec_at(exported_functions, ordinal); | |
| if (iree_vm_flatbuffer_strcmp( | |
| iree_vm_ExportFunctionDef_local_name(export_def), name) == 0) { | |
| out_function->ordinal = ordinal; | |
| return iree_ok_status(); | |
| } | |
| } | |
| } | |
| return iree_make_status(IREE_STATUS_NOT_FOUND, | |
| "function with the given name not found"); | |
| } | |
| static iree_status_t iree_vm_bytecode_location_format( | |
| int32_t location_ordinal, | |
| iree_vm_LocationTypeDef_union_vec_t location_table, | |
| iree_vm_source_location_format_flags_t flags, | |
| iree_string_builder_t* builder) { | |
| iree_vm_LocationTypeDef_union_t location = | |
| iree_vm_LocationTypeDef_union_vec_at(location_table, location_ordinal); | |
| switch (location.type) { | |
| default: | |
| case iree_vm_LocationTypeDef_NONE: { | |
| return iree_string_builder_append_cstring(builder, "[unknown]"); | |
| } | |
| case iree_vm_LocationTypeDef_CallSiteLocDef: { | |
| // NOTE: MLIR prints caller->callee, but in a stack trace we want the | |
| // upside-down callee->caller. | |
| iree_vm_CallSiteLocDef_table_t loc = | |
| (iree_vm_CallSiteLocDef_table_t)location.value; | |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_location_format( | |
| iree_vm_CallSiteLocDef_callee(loc), location_table, flags, builder)); | |
| IREE_RETURN_IF_ERROR( | |
| iree_string_builder_append_cstring(builder, "\n at ")); | |
| return iree_vm_bytecode_location_format( | |
| iree_vm_CallSiteLocDef_caller(loc), location_table, flags, builder); | |
| } | |
| case iree_vm_LocationTypeDef_FileLineColLocDef: { | |
| iree_vm_FileLineColLocDef_table_t loc = | |
| (iree_vm_FileLineColLocDef_table_t)location.value; | |
| flatbuffers_string_t filename = iree_vm_FileLineColLocDef_filename(loc); | |
| return iree_string_builder_append_format( | |
| builder, "%.*s:%d:%d", (int)flatbuffers_string_len(filename), | |
| filename, iree_vm_FileLineColLocDef_line(loc), | |
| iree_vm_FileLineColLocDef_column(loc)); | |
| } | |
| case iree_vm_LocationTypeDef_FusedLocDef: { | |
| iree_vm_FusedLocDef_table_t loc = | |
| (iree_vm_FusedLocDef_table_t)location.value; | |
| if (iree_vm_FusedLocDef_metadata_is_present(loc)) { | |
| flatbuffers_string_t metadata = iree_vm_FusedLocDef_metadata(loc); | |
| IREE_RETURN_IF_ERROR(iree_string_builder_append_format( | |
| builder, "<%.*s>", (int)flatbuffers_string_len(metadata), | |
| metadata)); | |
| } | |
| IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(builder, "[\n")); | |
| flatbuffers_int32_vec_t child_locs = iree_vm_FusedLocDef_locations(loc); | |
| for (size_t i = 0; i < flatbuffers_int32_vec_len(child_locs); ++i) { | |
| if (i == 0) { | |
| IREE_RETURN_IF_ERROR( | |
| iree_string_builder_append_cstring(builder, " ")); | |
| } else { | |
| IREE_RETURN_IF_ERROR( | |
| iree_string_builder_append_cstring(builder, ",\n ")); | |
| } | |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_location_format( | |
| flatbuffers_int32_vec_at(child_locs, i), location_table, flags, | |
| builder)); | |
| } | |
| IREE_RETURN_IF_ERROR( | |
| iree_string_builder_append_cstring(builder, "\n ]")); | |
| return iree_ok_status(); | |
| } | |
| case iree_vm_LocationTypeDef_NameLocDef: { | |
| iree_vm_NameLocDef_table_t loc = | |
| (iree_vm_NameLocDef_table_t)location.value; | |
| flatbuffers_string_t name = iree_vm_NameLocDef_name(loc); | |
| IREE_RETURN_IF_ERROR(iree_string_builder_append_format( | |
| builder, "\"%.*s\"", (int)flatbuffers_string_len(name), name)); | |
| if (iree_vm_NameLocDef_child_location_is_present(loc)) { | |
| IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(builder, "(")); | |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_location_format( | |
| iree_vm_NameLocDef_child_location(loc), location_table, flags, | |
| builder)); | |
| IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(builder, ")")); | |
| } | |
| return iree_ok_status(); | |
| } | |
| } | |
| } | |
| static iree_status_t iree_vm_bytecode_module_source_location_format( | |
| void* self, uint64_t data[2], iree_vm_source_location_format_flags_t flags, | |
| iree_string_builder_t* builder) { | |
| iree_vm_DebugDatabaseDef_table_t debug_database_def = | |
| (iree_vm_DebugDatabaseDef_table_t)self; | |
| iree_vm_FunctionSourceMapDef_table_t source_map_def = | |
| (iree_vm_FunctionSourceMapDef_table_t)data[0]; | |
| iree_vm_BytecodeLocationDef_vec_t locations = | |
| iree_vm_FunctionSourceMapDef_locations(source_map_def); | |
| iree_vm_source_offset_t source_offset = (iree_vm_source_offset_t)data[1]; | |
| size_t location_def_ordinal = | |
| iree_vm_BytecodeLocationDef_vec_scan_by_bytecode_offset( | |
| locations, (int32_t)source_offset); | |
| if (location_def_ordinal == -1) { | |
| return iree_status_from_code(IREE_STATUS_UNAVAILABLE); | |
| } | |
| iree_vm_BytecodeLocationDef_struct_t location_def = | |
| iree_vm_BytecodeLocationDef_vec_at(locations, location_def_ordinal); | |
| if (!location_def) { | |
| return iree_status_from_code(IREE_STATUS_UNAVAILABLE); | |
| } | |
| // Print source location stack trace. | |
| iree_vm_LocationTypeDef_union_vec_t location_table = | |
| iree_vm_DebugDatabaseDef_location_table_union(debug_database_def); | |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_location_format( | |
| location_def->location, location_table, flags, builder)); | |
| return iree_ok_status(); | |
| } | |
| static iree_status_t iree_vm_bytecode_module_resolve_source_location( | |
| void* self, iree_vm_stack_frame_t* frame, | |
| iree_vm_source_location_t* out_source_location) { | |
| // Get module debug database, if available. | |
| iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)self; | |
| iree_vm_BytecodeModuleDef_table_t module_def = module->def; | |
| iree_vm_DebugDatabaseDef_table_t debug_database_def = | |
| iree_vm_BytecodeModuleDef_debug_database(module_def); | |
| if (!debug_database_def) { | |
| return iree_status_from_code(IREE_STATUS_UNAVAILABLE); | |
| } | |
| // Map the (potentially) export ordinal into the internal function ordinal in | |
| // the function descriptor table. | |
| uint16_t ordinal; | |
| if (frame->function.linkage == IREE_VM_FUNCTION_LINKAGE_INTERNAL) { | |
| ordinal = frame->function.ordinal; | |
| } else { | |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_map_internal_ordinal( | |
| module, frame->function, &ordinal, NULL)); | |
| } | |
| // Lookup the source map for the function, if available. | |
| iree_vm_FunctionSourceMapDef_vec_t source_maps_vec = | |
| iree_vm_DebugDatabaseDef_functions(debug_database_def); | |
| iree_vm_FunctionSourceMapDef_table_t source_map_def = | |
| ordinal < iree_vm_FunctionSourceMapDef_vec_len(source_maps_vec) | |
| ? iree_vm_FunctionSourceMapDef_vec_at(source_maps_vec, ordinal) | |
| : NULL; | |
| if (!source_map_def) { | |
| return iree_status_from_code(IREE_STATUS_UNAVAILABLE); | |
| } | |
| // The source location stores the source map and PC and will perform the | |
| // actual lookup within the source map on demand. | |
| out_source_location->self = (void*)debug_database_def; | |
| out_source_location->data[0] = (uint64_t)source_map_def; | |
| out_source_location->data[1] = (uint64_t)frame->pc; | |
| out_source_location->format = iree_vm_bytecode_module_source_location_format; | |
| return iree_ok_status(); | |
| } | |
| // Lays out the nested tables within a |state| structure. | |
| // Returns the total size of the structure and all tables with padding applied. | |
| // |state| may be null if only the structure size is required for allocation. | |
| static iree_host_size_t iree_vm_bytecode_module_layout_state( | |
| iree_vm_BytecodeModuleDef_table_t module_def, | |
| iree_vm_bytecode_module_state_t* state) { | |
| iree_vm_ModuleStateDef_table_t module_state_def = | |
| iree_vm_BytecodeModuleDef_module_state(module_def); | |
| iree_host_size_t rwdata_storage_capacity = 0; | |
| iree_host_size_t global_ref_count = 0; | |
| if (module_state_def) { | |
| rwdata_storage_capacity = | |
| iree_vm_ModuleStateDef_global_bytes_capacity(module_state_def); | |
| global_ref_count = | |
| iree_vm_ModuleStateDef_global_ref_count(module_state_def); | |
| } | |
| iree_host_size_t rodata_ref_count = iree_vm_RodataSegmentDef_vec_len( | |
| iree_vm_BytecodeModuleDef_rodata_segments(module_def)); | |
| iree_host_size_t import_function_count = iree_vm_ImportFunctionDef_vec_len( | |
| iree_vm_BytecodeModuleDef_imported_functions(module_def)); | |
| uint8_t* base_ptr = (uint8_t*)state; | |
| iree_host_size_t offset = | |
| iree_host_align(sizeof(iree_vm_bytecode_module_state_t), 16); | |
| if (state) { | |
| state->rwdata_storage = | |
| iree_make_byte_span(base_ptr + offset, rwdata_storage_capacity); | |
| } | |
| offset += iree_host_align(rwdata_storage_capacity, 16); | |
| if (state) { | |
| state->global_ref_count = global_ref_count; | |
| state->global_ref_table = (iree_vm_ref_t*)(base_ptr + offset); | |
| } | |
| offset += iree_host_align(global_ref_count * sizeof(iree_vm_ref_t), 16); | |
| if (state) { | |
| state->rodata_ref_count = rodata_ref_count; | |
| state->rodata_ref_table = (iree_vm_buffer_t*)(base_ptr + offset); | |
| } | |
| offset += iree_host_align(rodata_ref_count * sizeof(iree_vm_buffer_t), 16); | |
| if (state) { | |
| state->import_count = import_function_count; | |
| state->import_table = (iree_vm_bytecode_import_t*)(base_ptr + offset); | |
| } | |
| offset += | |
| iree_host_align(import_function_count * sizeof(*state->import_table), 16); | |
| return offset; | |
| } | |
| static iree_status_t iree_vm_bytecode_module_alloc_state( | |
| void* self, iree_allocator_t allocator, | |
| iree_vm_module_state_t** out_module_state) { | |
| IREE_TRACE_ZONE_BEGIN(z0); | |
| IREE_ASSERT_ARGUMENT(out_module_state); | |
| *out_module_state = NULL; | |
| iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)self; | |
| iree_vm_BytecodeModuleDef_table_t module_def = module->def; | |
| // Compute the total size required (with padding) for the state structure. | |
| iree_host_size_t total_state_struct_size = | |
| iree_vm_bytecode_module_layout_state(module_def, NULL); | |
| // Allocate the storage for the structure and all its nested tables. | |
| iree_vm_bytecode_module_state_t* state = NULL; | |
| IREE_RETURN_AND_END_ZONE_IF_ERROR( | |
| z0, iree_allocator_malloc(allocator, total_state_struct_size, | |
| (void**)&state)); | |
| state->allocator = allocator; | |
| // Perform layout to get the pointers into the storage for each nested table. | |
| iree_vm_bytecode_module_layout_state(module_def, state); | |
| // Setup rodata segments to point directly at the FlatBuffer memory. | |
| const iree_host_size_t external_rodata_offset = | |
| iree_vm_bytecode_module_flatbuffer_rodata_offset(module->flatbuffer_data); | |
| iree_vm_RodataSegmentDef_vec_t rodata_segments = | |
| iree_vm_BytecodeModuleDef_rodata_segments(module_def); | |
| for (int i = 0; i < state->rodata_ref_count; ++i) { | |
| iree_vm_RodataSegmentDef_table_t segment = | |
| iree_vm_RodataSegmentDef_vec_at(rodata_segments, i); | |
| iree_byte_span_t byte_span = iree_byte_span_empty(); | |
| if (iree_vm_RodataSegmentDef_embedded_data_is_present(segment)) { | |
| // Data is embedded in the FlatBuffer. | |
| byte_span = iree_make_byte_span( | |
| (uint8_t*)iree_vm_RodataSegmentDef_embedded_data(segment), | |
| flatbuffers_uint8_vec_len( | |
| iree_vm_RodataSegmentDef_embedded_data(segment))); | |
| } else { | |
| // Data is concatenated with the FlatBuffer at some relative offset. | |
| // Note that we've already verified the referenced range is in bounds. | |
| byte_span = iree_make_byte_span( | |
| (uint8_t*)module->flatbuffer_data.data + external_rodata_offset + | |
| iree_vm_RodataSegmentDef_external_data_offset(segment), | |
| iree_vm_RodataSegmentDef_external_data_length(segment)); | |
| } | |
| iree_vm_buffer_t* ref = &state->rodata_ref_table[i]; | |
| iree_vm_buffer_initialize(IREE_VM_BUFFER_ACCESS_ORIGIN_MODULE, byte_span, | |
| iree_allocator_null(), ref); | |
| } | |
| *out_module_state = (iree_vm_module_state_t*)state; | |
| IREE_TRACE_ZONE_END(z0); | |
| return iree_ok_status(); | |
| } | |
| static void iree_vm_bytecode_module_free_state( | |
| void* self, iree_vm_module_state_t* module_state) { | |
| if (!module_state) return; | |
| IREE_TRACE_ZONE_BEGIN(z0); | |
| iree_vm_bytecode_module_state_t* state = | |
| (iree_vm_bytecode_module_state_t*)module_state; | |
| // Release remaining global references. | |
| for (int i = 0; i < state->global_ref_count; ++i) { | |
| iree_vm_ref_release(&state->global_ref_table[i]); | |
| } | |
| // Ensure all rodata references are unused and deinitialized. | |
| for (int i = 0; i < state->rodata_ref_count; ++i) { | |
| iree_vm_buffer_t* ref = &state->rodata_ref_table[i]; | |
| iree_vm_buffer_deinitialize(ref); | |
| } | |
| iree_allocator_free(state->allocator, module_state); | |
| IREE_TRACE_ZONE_END(z0); | |
| } | |
| static iree_status_t iree_vm_bytecode_module_resolve_import( | |
| void* self, iree_vm_module_state_t* module_state, iree_host_size_t ordinal, | |
| const iree_vm_function_t* function, | |
| const iree_vm_function_signature_t* signature) { | |
| IREE_ASSERT_ARGUMENT(module_state); | |
| iree_vm_bytecode_module_state_t* state = | |
| (iree_vm_bytecode_module_state_t*)module_state; | |
| if (ordinal >= state->import_count) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "import ordinal out of range (0 < %zu < %zu)", | |
| ordinal, state->import_count); | |
| } | |
| iree_vm_bytecode_import_t* import = &state->import_table[ordinal]; | |
| import->function = *function; | |
| // Split up arguments/results into fragments so that we can avoid scanning | |
| // during calling. | |
| IREE_RETURN_IF_ERROR(iree_vm_function_call_get_cconv_fragments( | |
| signature, &import->arguments, &import->results)); | |
| // Precalculate bytes required to marshal argument/results across the ABI | |
| // boundary. | |
| iree_host_size_t argument_buffer_size = 0; | |
| iree_host_size_t result_buffer_size = 0; | |
| if (!iree_vm_function_call_is_variadic_cconv(import->arguments)) { | |
| // NOTE: variadic types don't support precalculation and the vm.call.import | |
| // dispatch code will handle calculating it per-call. | |
| IREE_RETURN_IF_ERROR(iree_vm_function_call_compute_cconv_fragment_size( | |
| import->arguments, /*segment_size_list=*/NULL, &argument_buffer_size)); | |
| } | |
| IREE_RETURN_IF_ERROR(iree_vm_function_call_compute_cconv_fragment_size( | |
| import->results, /*segment_size_list=*/NULL, &result_buffer_size)); | |
| if (argument_buffer_size > 16 * 1024 || result_buffer_size > 16 * 1024) { | |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, | |
| "ABI marshaling buffer overflow on import %zu", | |
| ordinal); | |
| } | |
| import->argument_buffer_size = (uint16_t)argument_buffer_size; | |
| import->result_buffer_size = (uint16_t)result_buffer_size; | |
| return iree_ok_status(); | |
| } | |
| static iree_status_t IREE_API_PTR iree_vm_bytecode_module_notify( | |
| void* self, iree_vm_module_state_t* module_state, iree_vm_signal_t signal) { | |
| return iree_ok_status(); | |
| } | |
| static iree_status_t iree_vm_bytecode_module_begin_call( | |
| void* self, iree_vm_stack_t* stack, const iree_vm_function_call_t* call, | |
| iree_vm_execution_result_t* out_result) { | |
| // NOTE: any work here adds directly to the invocation time. Avoid doing too | |
| // much work or touching too many unlikely-to-be-cached structures (such as | |
| // walking the FlatBuffer, which may cause page faults). | |
| IREE_TRACE_ZONE_BEGIN(z0); | |
| IREE_ASSERT_ARGUMENT(out_result); | |
| memset(out_result, 0, sizeof(iree_vm_execution_result_t)); | |
| // Map the (potentially) export ordinal into the internal function ordinal in | |
| // the function descriptor table. | |
| iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)self; | |
| uint16_t ordinal = 0; | |
| iree_vm_FunctionSignatureDef_table_t signature_def = NULL; | |
| IREE_RETURN_AND_END_ZONE_IF_ERROR( | |
| z0, iree_vm_bytecode_map_internal_ordinal(module, call->function, | |
| &ordinal, &signature_def)); | |
| // Grab calling convention string. This is not great as we are guaranteed to | |
| // have a bunch of cache misses, but without putting it on the descriptor | |
| // (which would duplicate data and slow down normal intra-module calls) | |
| // there's not a good way around it. In the grand scheme of things users | |
| // should be keeping their calls across this boundary relatively fat (compared | |
| // to the real work they do), so this only needs to be fast enough to blend | |
| // into the noise. Similar to JNI, P/Invoke, etc you don't want to have | |
| // imports that cost less to execute than the marshaling overhead (dozens to | |
| // hundreds of instructions). | |
| flatbuffers_string_t calling_convention = | |
| signature_def | |
| ? iree_vm_FunctionSignatureDef_calling_convention(signature_def) | |
| : 0; | |
| iree_vm_function_signature_t signature; | |
| memset(&signature, 0, sizeof(signature)); | |
| signature.calling_convention.data = calling_convention; | |
| signature.calling_convention.size = | |
| flatbuffers_string_len(calling_convention); | |
| iree_string_view_t cconv_arguments = iree_string_view_empty(); | |
| iree_string_view_t cconv_results = iree_string_view_empty(); | |
| IREE_RETURN_AND_END_ZONE_IF_ERROR( | |
| z0, iree_vm_function_call_get_cconv_fragments( | |
| &signature, &cconv_arguments, &cconv_results)); | |
| // Jump into the dispatch routine to execute bytecode until the function | |
| // either returns (synchronous) or yields (asynchronous). | |
| iree_status_t status = iree_vm_bytecode_dispatch( | |
| stack, module, call, cconv_arguments, cconv_results, out_result); | |
| IREE_TRACE_ZONE_END(z0); | |
| return status; | |
| } | |
| IREE_API_EXPORT iree_status_t iree_vm_bytecode_module_create( | |
| iree_const_byte_span_t flatbuffer_data, | |
| iree_allocator_t flatbuffer_allocator, iree_allocator_t allocator, | |
| iree_vm_module_t** out_module) { | |
| IREE_TRACE_ZONE_BEGIN(z0); | |
| IREE_ASSERT_ARGUMENT(out_module); | |
| *out_module = NULL; | |
| IREE_TRACE_ZONE_BEGIN_NAMED(z1, "iree_vm_bytecode_module_flatbuffer_verify"); | |
| iree_status_t status = | |
| iree_vm_bytecode_module_flatbuffer_verify(flatbuffer_data); | |
| if (!iree_status_is_ok(status)) { | |
| IREE_TRACE_ZONE_END(z1); | |
| IREE_TRACE_ZONE_END(z0); | |
| return status; | |
| } | |
| IREE_TRACE_ZONE_END(z1); | |
| iree_vm_BytecodeModuleDef_table_t module_def = | |
| iree_vm_BytecodeModuleDef_as_root(flatbuffer_data.data); | |
| if (!module_def) { | |
| IREE_TRACE_ZONE_END(z0); | |
| return iree_make_status( | |
| IREE_STATUS_INVALID_ARGUMENT, | |
| "failed getting root from flatbuffer; expected identifier " | |
| "'" iree_vm_BytecodeModuleDef_file_identifier "' not found"); | |
| } | |
| iree_vm_TypeDef_vec_t type_defs = iree_vm_BytecodeModuleDef_types(module_def); | |
| size_t type_table_size = | |
| iree_vm_TypeDef_vec_len(type_defs) * sizeof(iree_vm_type_def_t); | |
| iree_vm_bytecode_module_t* module = NULL; | |
| IREE_RETURN_AND_END_ZONE_IF_ERROR( | |
| z0, iree_allocator_malloc(allocator, sizeof(*module) + type_table_size, | |
| (void**)&module)); | |
| module->allocator = allocator; | |
| iree_vm_FunctionDescriptor_vec_t function_descriptors = | |
| iree_vm_BytecodeModuleDef_function_descriptors(module_def); | |
| module->function_descriptor_count = | |
| iree_vm_FunctionDescriptor_vec_len(function_descriptors); | |
| module->function_descriptor_table = function_descriptors; | |
| flatbuffers_uint8_vec_t bytecode_data = | |
| iree_vm_BytecodeModuleDef_bytecode_data(module_def); | |
| module->bytecode_data = iree_make_const_byte_span( | |
| bytecode_data, flatbuffers_uint8_vec_len(bytecode_data)); | |
| module->flatbuffer_data = flatbuffer_data; | |
| module->flatbuffer_allocator = flatbuffer_allocator; | |
| module->def = module_def; | |
| module->type_count = iree_vm_TypeDef_vec_len(type_defs); | |
| iree_status_t resolve_status = | |
| iree_vm_bytecode_module_resolve_types(type_defs, module->type_table); | |
| if (!iree_status_is_ok(resolve_status)) { | |
| iree_allocator_free(allocator, module); | |
| IREE_TRACE_ZONE_END(z0); | |
| return resolve_status; | |
| } | |
| iree_vm_module_initialize(&module->interface, module); | |
| module->interface.destroy = iree_vm_bytecode_module_destroy; | |
| module->interface.name = iree_vm_bytecode_module_name; | |
| module->interface.signature = iree_vm_bytecode_module_signature; | |
| module->interface.get_function = iree_vm_bytecode_module_get_function; | |
| module->interface.lookup_function = iree_vm_bytecode_module_lookup_function; | |
| #if IREE_VM_BACKTRACE_ENABLE | |
| module->interface.resolve_source_location = | |
| iree_vm_bytecode_module_resolve_source_location; | |
| #endif // IREE_VM_BACKTRACE_ENABLE | |
| module->interface.alloc_state = iree_vm_bytecode_module_alloc_state; | |
| module->interface.free_state = iree_vm_bytecode_module_free_state; | |
| module->interface.resolve_import = iree_vm_bytecode_module_resolve_import; | |
| module->interface.notify = iree_vm_bytecode_module_notify; | |
| module->interface.begin_call = iree_vm_bytecode_module_begin_call; | |
| module->interface.get_function_reflection_attr = | |
| iree_vm_bytecode_module_get_function_reflection_attr; | |
| *out_module = &module->interface; | |
| IREE_TRACE_ZONE_END(z0); | |
| return iree_ok_status(); | |
| } |