| // Copyright 2023 The IREE Authors |
| // |
| // Licensed under the Apache License v2.0 with LLVM Exceptions. |
| // See https://llvm.org/LICENSE.txt for license information. |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |
| |
| #include "iree/vm/bytecode/verifier.h" |
| |
| #include "iree/base/internal/math.h" |
| #include "iree/vm/bytecode/utils/block_list.h" |
| #include "iree/vm/bytecode/utils/features.h" |
| |
| //===----------------------------------------------------------------------===// |
| // Module metadata verification |
| //===----------------------------------------------------------------------===// |
| |
| iree_status_t iree_vm_bytecode_module_flatbuffer_verify( |
| iree_const_byte_span_t archive_contents, |
| iree_const_byte_span_t flatbuffer_contents, |
| iree_host_size_t archive_rodata_offset) { |
| // Run flatcc generated verification. This ensures all pointers are in-bounds |
| // and that we can safely walk the file, but not that the actual contents of |
| // the FlatBuffer meet our expectations. |
| int verify_ret = iree_vm_BytecodeModuleDef_verify_as_root( |
| flatbuffer_contents.data, flatbuffer_contents.data_length); |
| if (verify_ret != flatcc_verify_ok) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "FlatBuffer verification failed: %s", |
| flatcc_verify_error_string(verify_ret)); |
| } |
| |
| iree_vm_BytecodeModuleDef_table_t module_def = |
| iree_vm_BytecodeModuleDef_as_root(flatbuffer_contents.data); |
| |
| const iree_vm_FeatureBits_enum_t available_features = |
| iree_vm_bytecode_available_features(); |
| const iree_vm_FeatureBits_enum_t required_features = |
| iree_vm_BytecodeModuleDef_requirements(module_def); |
| IREE_RETURN_IF_ERROR(iree_vm_check_feature_mismatch( |
| __FILE__, __LINE__, required_features, available_features)); |
| |
| flatbuffers_string_t name = iree_vm_BytecodeModuleDef_name(module_def); |
| if (!flatbuffers_string_len(name)) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "module missing name field"); |
| } |
| |
| iree_vm_TypeDef_vec_t types = iree_vm_BytecodeModuleDef_types(module_def); |
| for (size_t i = 0; i < iree_vm_TypeDef_vec_len(types); ++i) { |
| iree_vm_TypeDef_table_t type_def = iree_vm_TypeDef_vec_at(types, i); |
| if (!type_def) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "types[%zu] missing body", i); |
| } |
| flatbuffers_string_t full_name = iree_vm_TypeDef_full_name(type_def); |
| if (flatbuffers_string_len(full_name) <= 0) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "types[%zu] missing name", i); |
| } |
| } |
| |
| iree_vm_RodataSegmentDef_vec_t rodata_segments = |
| iree_vm_BytecodeModuleDef_rodata_segments(module_def); |
| for (size_t i = 0; i < iree_vm_RodataSegmentDef_vec_len(rodata_segments); |
| ++i) { |
| iree_vm_RodataSegmentDef_table_t segment = |
| iree_vm_RodataSegmentDef_vec_at(rodata_segments, i); |
| if (iree_vm_RodataSegmentDef_embedded_data_is_present(segment)) { |
| continue; // embedded data is verified by FlatBuffers |
| } |
| uint64_t segment_offset = |
| iree_vm_RodataSegmentDef_external_data_offset(segment); |
| uint64_t segment_length = |
| iree_vm_RodataSegmentDef_external_data_length(segment); |
| uint64_t segment_end = |
| archive_rodata_offset + segment_offset + segment_length; |
| if (segment_end > archive_contents.data_length) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "rodata[%zu] external reference out of range", i); |
| } |
| } |
| |
| iree_vm_ModuleDependencyDef_vec_t dependencies = |
| iree_vm_BytecodeModuleDef_dependencies(module_def); |
| for (size_t i = 0; i < iree_vm_ModuleDependencyDef_vec_len(dependencies); |
| ++i) { |
| iree_vm_ModuleDependencyDef_table_t dependency_def = |
| iree_vm_ModuleDependencyDef_vec_at(dependencies, i); |
| flatbuffers_string_t module_name = |
| iree_vm_ModuleDependencyDef_name(dependency_def); |
| if (flatbuffers_string_len(module_name) == 0) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "dependencies[%zu] has no module name", i); |
| } |
| } |
| |
| iree_vm_ImportFunctionDef_vec_t imported_functions = |
| iree_vm_BytecodeModuleDef_imported_functions(module_def); |
| iree_vm_ExportFunctionDef_vec_t exported_functions = |
| iree_vm_BytecodeModuleDef_exported_functions(module_def); |
| iree_vm_FunctionSignatureDef_vec_t function_signatures = |
| iree_vm_BytecodeModuleDef_function_signatures(module_def); |
| iree_vm_FunctionDescriptor_vec_t function_descriptors = |
| iree_vm_BytecodeModuleDef_function_descriptors(module_def); |
| |
| for (size_t i = 0; |
| i < iree_vm_FunctionSignatureDef_vec_len(function_signatures); ++i) { |
| iree_vm_FunctionSignatureDef_table_t function_signature = |
| iree_vm_FunctionSignatureDef_vec_at(function_signatures, i); |
| if (!function_signature) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "function_signatures[%zu] missing body", i); |
| } |
| } |
| |
| for (size_t i = 0; i < iree_vm_ImportFunctionDef_vec_len(imported_functions); |
| ++i) { |
| iree_vm_ImportFunctionDef_table_t import_def = |
| iree_vm_ImportFunctionDef_vec_at(imported_functions, i); |
| if (!import_def) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "imports[%zu] missing body", i); |
| } |
| flatbuffers_string_t full_name = |
| iree_vm_ImportFunctionDef_full_name(import_def); |
| if (!flatbuffers_string_len(full_name)) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "imports[%zu] missing full_name", i); |
| } |
| } |
| |
| if (iree_vm_FunctionSignatureDef_vec_len(function_signatures) != |
| iree_vm_FunctionDescriptor_vec_len(function_descriptors)) { |
| return iree_make_status( |
| IREE_STATUS_INVALID_ARGUMENT, |
| "function signature and descriptor table length mismatch (%zu vs %zu)", |
| iree_vm_FunctionSignatureDef_vec_len(function_signatures), |
| iree_vm_FunctionDescriptor_vec_len(function_descriptors)); |
| } |
| |
| for (size_t i = 0; i < iree_vm_ExportFunctionDef_vec_len(exported_functions); |
| ++i) { |
| iree_vm_ExportFunctionDef_table_t export_def = |
| iree_vm_ExportFunctionDef_vec_at(exported_functions, i); |
| if (!export_def) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "exports[%zu] missing body", i); |
| } |
| flatbuffers_string_t local_name = |
| iree_vm_ExportFunctionDef_local_name(export_def); |
| if (!flatbuffers_string_len(local_name)) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "exports[%zu] missing local_name", i); |
| } |
| iree_host_size_t internal_ordinal = |
| iree_vm_ExportFunctionDef_internal_ordinal(export_def); |
| if (internal_ordinal >= |
| iree_vm_FunctionDescriptor_vec_len(function_descriptors)) { |
| return iree_make_status( |
| IREE_STATUS_INVALID_ARGUMENT, |
| "exports[%zu] internal_ordinal out of bounds (0 < %" PRIhsz " < %zu)", |
| i, internal_ordinal, |
| iree_vm_FunctionDescriptor_vec_len(function_descriptors)); |
| } |
| } |
| |
| // Verify that we can properly handle the bytecode embedded in the module. |
| // We require that major versions match and allow loading of older minor |
| // versions (we keep changes backwards-compatible). |
| const uint32_t bytecode_version = |
| iree_vm_BytecodeModuleDef_bytecode_version(module_def); |
| const uint32_t bytecode_version_major = bytecode_version >> 16; |
| const uint32_t bytecode_version_minor = bytecode_version & 0xFFFF; |
| if ((bytecode_version_major != IREE_VM_BYTECODE_VERSION_MAJOR) || |
| (bytecode_version_minor > IREE_VM_BYTECODE_VERSION_MINOR)) { |
| return iree_make_status( |
| IREE_STATUS_INVALID_ARGUMENT, |
| "bytecode version mismatch; runtime supports %d.%d, module has %d.%d", |
| IREE_VM_BYTECODE_VERSION_MAJOR, IREE_VM_BYTECODE_VERSION_MINOR, |
| bytecode_version_major, bytecode_version_minor); |
| } |
| |
| flatbuffers_uint8_vec_t bytecode_data = |
| iree_vm_BytecodeModuleDef_bytecode_data(module_def); |
| for (size_t i = 0; |
| i < iree_vm_FunctionDescriptor_vec_len(function_descriptors); ++i) { |
| iree_vm_FunctionDescriptor_struct_t function_descriptor = |
| iree_vm_FunctionDescriptor_vec_at(function_descriptors, i); |
| if (function_descriptor->block_count == 0) { |
| return iree_make_status( |
| IREE_STATUS_INVALID_ARGUMENT, |
| "functions[%zu] descriptor block count is 0; " |
| "functions must have at least 1 block, expected %d", |
| i, function_descriptor->block_count); |
| } |
| if (function_descriptor->bytecode_length == 0) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "functions[%zu] descriptor bytecode reports 0 " |
| "length; functions must have at least one block", |
| i); |
| } |
| if (function_descriptor->bytecode_offset < 0 || |
| function_descriptor->bytecode_offset + |
| function_descriptor->bytecode_length > |
| flatbuffers_uint8_vec_len(bytecode_data)) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "functions[%zu] descriptor bytecode span out of " |
| "range (0 < %d < %zu)", |
| i, function_descriptor->bytecode_offset, |
| flatbuffers_uint8_vec_len(bytecode_data)); |
| } |
| if (function_descriptor->i32_register_count > IREE_I32_REGISTER_COUNT || |
| function_descriptor->ref_register_count > IREE_REF_REGISTER_COUNT) { |
| return iree_make_status( |
| IREE_STATUS_INVALID_ARGUMENT, |
| "functions[%zu] descriptor register count out of range", i); |
| } |
| } |
| |
| return iree_ok_status(); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Function verification |
| //===----------------------------------------------------------------------===// |
| |
| // State used during verification of a function. |
| typedef struct iree_vm_bytecode_verify_state_t { |
| // Within a block (encountered a block marker and not yet a terminator). |
| uint32_t in_block : 1; |
| |
| // Maximum valid register ordinals. |
| uint32_t i32_register_count; |
| uint32_t ref_register_count; |
| |
| // Parsed argument and result cconv fragments. |
| iree_string_view_t cconv_arguments; |
| iree_string_view_t cconv_results; |
| |
| // All block branch points. |
| iree_vm_bytecode_block_list_t block_list; |
| |
| // Quick lookups of flatbuffer properties. |
| const iree_vm_ImportFunctionDef_vec_t imported_functions; |
| const iree_vm_ExportFunctionDef_vec_t exported_functions; |
| const iree_vm_FunctionSignatureDef_vec_t function_signatures; |
| const iree_vm_FunctionDescriptor_vec_t function_descriptors; |
| iree_host_size_t rodata_storage_size; |
| iree_host_size_t rodata_ref_count; |
| iree_host_size_t rwdata_storage_size; |
| iree_host_size_t global_ref_count; |
| } iree_vm_bytecode_verify_state_t; |
| |
| // Parses the cconv fragments from the given |signature_def|. |
| static iree_status_t iree_vm_bytecode_function_get_cconv_fragments( |
| iree_vm_FunctionSignatureDef_table_t signature_def, |
| iree_string_view_t* out_arguments, iree_string_view_t* out_results); |
| |
| // Verifies that the function has storage for the declared arguments. |
| static iree_status_t iree_vm_bytecode_function_verify_arguments( |
| const iree_vm_bytecode_verify_state_t* verify_state); |
| |
| // Verifies a single operation at |pc| in the function |bytecode_data|. |
| // Returns an error if the op is invalid and otherwise sets |out_next_pc| to the |
| // program counter immediately following the op (which may be the end of data!). |
| static iree_status_t iree_vm_bytecode_function_verify_bytecode_op( |
| iree_vm_bytecode_module_t* module, |
| iree_vm_bytecode_verify_state_t* verify_state, |
| iree_vm_FunctionSignatureDef_table_t function_signature, |
| iree_vm_FunctionDescriptor_struct_t function_descriptor, |
| iree_const_byte_span_t bytecode_data, uint32_t pc, uint32_t max_pc, |
| uint32_t* out_next_pc); |
| |
| // NOTE: by the time this is called we have the module created and can assume |
| // all information on it has been verified. The only thing this verifies is |
| // function bytecode and capabilities! |
| iree_status_t iree_vm_bytecode_function_verify( |
| iree_vm_bytecode_module_t* module, uint16_t function_ordinal, |
| iree_allocator_t scratch_allocator) { |
| if (function_ordinal >= module->function_descriptor_count) { |
| return iree_make_status(IREE_STATUS_OUT_OF_RANGE, |
| "invalid function ordinal"); |
| } |
| iree_vm_FunctionSignatureDef_table_t function_signature_def = |
| iree_vm_FunctionSignatureDef_vec_at( |
| iree_vm_BytecodeModuleDef_function_signatures(module->def), |
| function_ordinal); |
| const iree_vm_FunctionDescriptor_t* function_descriptor = |
| &module->function_descriptor_table[function_ordinal]; |
| |
| const iree_vm_FeatureBits_enum_t available_features = |
| iree_vm_bytecode_available_features(); |
| const iree_vm_FeatureBits_enum_t required_features = |
| function_descriptor->requirements; |
| IREE_RETURN_IF_ERROR(iree_vm_check_feature_mismatch( |
| __FILE__, __LINE__, required_features, available_features)); |
| |
| if (function_descriptor->block_count == 0) { |
| return iree_make_status( |
| IREE_STATUS_OUT_OF_RANGE, |
| "no blocks defined; functions must have at least one block"); |
| } |
| |
| // State used during verification. |
| iree_vm_bytecode_verify_state_t verify_state = { |
| .in_block = 0, |
| .imported_functions = |
| iree_vm_BytecodeModuleDef_imported_functions(module->def), |
| .exported_functions = |
| iree_vm_BytecodeModuleDef_exported_functions(module->def), |
| .function_signatures = |
| iree_vm_BytecodeModuleDef_function_signatures(module->def), |
| .function_descriptors = |
| iree_vm_BytecodeModuleDef_function_descriptors(module->def), |
| .rodata_storage_size = 0, |
| .rodata_ref_count = 0, |
| .rwdata_storage_size = 0, |
| .global_ref_count = 0, |
| }; |
| |
| // NOTE: these must be consistent with iree_vm_bytecode_module_layout_state. |
| verify_state.rodata_storage_size = 0; |
| verify_state.rodata_ref_count = iree_vm_RodataSegmentDef_vec_len( |
| iree_vm_BytecodeModuleDef_rodata_segments(module->def)); |
| iree_vm_ModuleStateDef_table_t module_state_def = |
| iree_vm_BytecodeModuleDef_module_state(module->def); |
| if (module_state_def) { |
| verify_state.rwdata_storage_size = |
| iree_vm_ModuleStateDef_global_bytes_capacity(module_state_def); |
| verify_state.global_ref_count = |
| iree_vm_ModuleStateDef_global_ref_count(module_state_def); |
| } |
| |
| // Ensure the register storage (rounded to the nearest power of 2) won't |
| // exceed the maximum allowed registers. |
| verify_state.i32_register_count = iree_math_round_up_to_pow2_u32( |
| VMMAX(1, function_descriptor->i32_register_count)); |
| verify_state.ref_register_count = iree_math_round_up_to_pow2_u32( |
| VMMAX(1, function_descriptor->ref_register_count)); |
| if (IREE_UNLIKELY(verify_state.i32_register_count > IREE_I32_REGISTER_MASK) || |
| IREE_UNLIKELY(verify_state.ref_register_count > IREE_REF_REGISTER_MASK)) { |
| // Register count overflow. A valid compiler should never produce files that |
| // hit this. |
| return iree_make_status(IREE_STATUS_RESOURCE_EXHAUSTED, |
| "register count overflow"); |
| } |
| |
| // Grab the cconv fragments declaring the arguments/results of the function. |
| verify_state.cconv_arguments = iree_string_view_empty(); |
| verify_state.cconv_results = iree_string_view_empty(); |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_get_cconv_fragments( |
| function_signature_def, &verify_state.cconv_arguments, |
| &verify_state.cconv_results)); |
| |
| // Verify there is storage for passed arguments. |
| IREE_RETURN_IF_ERROR( |
| iree_vm_bytecode_function_verify_arguments(&verify_state)); |
| |
| // NOTE: module verification ensures the function descriptor has a valid |
| // bytecode range so we can assume that's true here. |
| IREE_ASSERT(function_descriptor->bytecode_length > 0); |
| IREE_ASSERT(function_descriptor->bytecode_offset >= 0); |
| IREE_ASSERT(function_descriptor->bytecode_offset + |
| function_descriptor->bytecode_length <= |
| module->bytecode_data.data_length); |
| iree_const_byte_span_t bytecode_data = iree_make_const_byte_span( |
| module->bytecode_data.data + function_descriptor->bytecode_offset, |
| function_descriptor->bytecode_length); |
| const uint32_t max_pc = (uint32_t)function_descriptor->bytecode_length; |
| |
| // Reserve the block list. As we walk the bytecode we'll declare/define blocks |
| // and then afterward verify all were found. |
| IREE_ASSERT(function_descriptor->block_count > 0); |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_block_list_initialize( |
| function_descriptor->block_count, scratch_allocator, |
| &verify_state.block_list)); |
| |
| // Perform bytecode verification by performing a single-pass walk of all |
| // function bytecode. |
| iree_status_t status = iree_ok_status(); |
| for (uint32_t pc = 0; pc < bytecode_data.data_length - 1;) { |
| uint32_t start_pc = pc; |
| status = iree_vm_bytecode_function_verify_bytecode_op( |
| module, &verify_state, function_signature_def, function_descriptor, |
| bytecode_data, start_pc, max_pc, &pc); |
| if (!iree_status_is_ok(status)) { |
| #if IREE_STATUS_MODE |
| // To get a useful source location we have to ask the main module; the |
| // base function table may only contain public symbols and not any |
| // internal ones. |
| iree_string_view_t module_name = iree_vm_module_name(&module->interface); |
| iree_vm_function_t function = {0}; |
| iree_status_ignore(iree_vm_module_lookup_function_by_ordinal( |
| &module->interface, IREE_VM_FUNCTION_LINKAGE_INTERNAL, |
| function_ordinal, &function)); |
| iree_string_view_t function_name = iree_vm_function_name(&function); |
| if (!iree_string_view_is_empty(function_name)) { |
| status = iree_status_annotate_f(status, "at %.*s.%.*s+%08X", |
| (int)module_name.size, module_name.data, |
| (int)function_name.size, |
| function_name.data, start_pc); |
| } else { |
| status = iree_status_annotate_f(status, "at %.*s@%u+%08X", |
| (int)module_name.size, module_name.data, |
| function_ordinal, start_pc); |
| } |
| #endif // IREE_STATUS_MODE |
| break; |
| } |
| } |
| |
| // Ensure there was a terminator. |
| if (iree_status_is_ok(status) && verify_state.in_block) { |
| status = iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "function missing terminator in the last block"); |
| } |
| |
| // Verify all blocks are defined and have proper markers. |
| if (iree_status_is_ok(status)) { |
| status = iree_vm_bytecode_block_list_verify(&verify_state.block_list, |
| bytecode_data); |
| } |
| |
| iree_vm_bytecode_block_list_deinitialize(&verify_state.block_list, |
| scratch_allocator); |
| |
| return status; |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Utilities matching the tablegen op encoding scheme |
| //===----------------------------------------------------------------------===// |
| // These utilities match the VM_Enc* statements in VMBase.td 1:1, allowing us |
| // to have the inverse of the encoding which make things easier to read. |
| // |
| // Each macro will increment the pc by the number of bytes read and as such must |
| // be called in the same order the values are encoded. |
| |
| // Bails if the |pc| exceeds the |max_pc|. |
| #define IREE_VM_VERIFY_PC_RANGE(pc, max_pc) \ |
| if (IREE_UNLIKELY((pc) > (max_pc))) { \ |
| return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ |
| "bytecode data overrun trying to parsing op at " \ |
| "%08X (%u) of %u available bytes", \ |
| (uint32_t)(pc), (uint32_t)(pc), \ |
| (uint32_t)(max_pc)); \ |
| } |
| |
| // Bails if the function doesn't have the given |required_features| declared. |
| #define IREE_VM_VERIFY_REQUIREMENT(required_features) \ |
| if (IREE_UNLIKELY(!iree_all_bits_set(function_descriptor->requirements, \ |
| (required_features)))) { \ |
| return iree_vm_check_feature_mismatch(__FILE__, __LINE__, \ |
| (required_features), \ |
| function_descriptor->requirements); \ |
| } |
| |
| // Bails if the register ordinal for the given register type is out of bounds. |
| #define IREE_VM_VERIFY_REG_ORDINAL(name) \ |
| IREE_VM_VERIFY_PC_RANGE(pc + IREE_REGISTER_ORDINAL_SIZE, max_pc); \ |
| const uint32_t name = OP_I16(0); |
| #define IREE_VM_VERIFY_REG_ORDINAL_X32(ordinal, category) \ |
| if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_TYPE_BIT) != 0)) { \ |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ |
| category \ |
| " register required but ref register %u provided", \ |
| (ordinal)); \ |
| } else if (IREE_UNLIKELY((ordinal) >= verify_state->i32_register_count)) { \ |
| return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ |
| category " register ordinal %u out of range %u", \ |
| (ordinal), verify_state->i32_register_count); \ |
| } |
| #define IREE_VM_VERIFY_REG_ORDINAL_X64(ordinal, category) \ |
| if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_TYPE_BIT) != 0)) { \ |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ |
| category \ |
| " register required but ref register %u provided", \ |
| (ordinal)); \ |
| } else if (IREE_UNLIKELY((ordinal & 1) != 0)) { \ |
| return iree_make_status( \ |
| IREE_STATUS_INVALID_ARGUMENT, \ |
| category " register ordinal %u not 8-byte aligned", (ordinal)); \ |
| } else if (IREE_UNLIKELY((ordinal) + 1 >= \ |
| verify_state->i32_register_count)) { \ |
| return iree_make_status( \ |
| IREE_STATUS_OUT_OF_RANGE, \ |
| category " register ordinal %u:%u out of range %u", (ordinal), \ |
| (ordinal) + 1, verify_state->i32_register_count); \ |
| } |
| #define IREE_VM_VERIFY_REG_I32(ordinal) \ |
| IREE_VM_VERIFY_REG_ORDINAL_X32(ordinal, "i32"); |
| #define IREE_VM_VERIFY_REG_I64(ordinal) \ |
| IREE_VM_VERIFY_REG_ORDINAL_X64(ordinal, "i64"); |
| #define IREE_VM_VERIFY_REG_F32(ordinal) \ |
| IREE_VM_VERIFY_REG_ORDINAL_X32(ordinal, "f32"); |
| #define IREE_VM_VERIFY_REG_F64(ordinal) \ |
| IREE_VM_VERIFY_REG_ORDINAL_X64(ordinal, "f64"); |
| #define IREE_VM_VERIFY_REG_REF(ordinal) \ |
| if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_TYPE_BIT) == 0)) { \ |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ |
| "ref register required but non-ref %u provided", \ |
| (ordinal)); \ |
| } else if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_MASK) >= \ |
| verify_state->ref_register_count)) { \ |
| return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ |
| "ref register ordinal %u out of range %u", \ |
| (ordinal), verify_state->ref_register_count); \ |
| } |
| #define IREE_VM_VERIFY_REG_ANY(ordinal) \ |
| if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_TYPE_BIT) == 0)) { \ |
| } else { \ |
| } |
| |
| #define VM_VerifyConstI8(name) \ |
| IREE_VM_VERIFY_PC_RANGE(pc + 1, max_pc); \ |
| uint8_t name = OP_I8(0); \ |
| (void)(name); \ |
| ++pc; |
| #define VM_VerifyConstI32(name) \ |
| IREE_VM_VERIFY_PC_RANGE(pc + 4, max_pc); \ |
| uint32_t name = OP_I32(0); \ |
| (void)(name); \ |
| pc += 4; |
| #define VM_VerifyConstI64(name) \ |
| IREE_VM_VERIFY_PC_RANGE(pc + 8, max_pc); \ |
| uint64_t name = OP_I64(0); \ |
| (void)(name); \ |
| pc += 8; |
| #define VM_VerifyConstF32(name) \ |
| IREE_VM_VERIFY_PC_RANGE(pc + 4, max_pc); \ |
| float name = OP_F32(0); \ |
| (void)(name); \ |
| pc += 4; |
| #define VM_VerifyConstF64(name) \ |
| IREE_VM_VERIFY_PC_RANGE(pc + 8, max_pc); \ |
| double name = OP_F64(0); \ |
| (void)(name); \ |
| pc += 8; |
| |
| #define VM_VerifyFuncAttr(name) VM_VerifyConstI32(name) |
| #define VM_IsImportOrdinal(name) (((name)&0x80000000u) != 0) |
| #define VM_UnmaskImportOrdinal(name) name &= ~0x80000000u |
| #define VM_VerifyImportOrdinal(name) \ |
| if (IREE_UNLIKELY((name) >= iree_vm_ImportFunctionDef_vec_len( \ |
| verify_state->imported_functions))) { \ |
| return iree_make_status( \ |
| IREE_STATUS_INVALID_ARGUMENT, "import ordinal %u out of range %zu", \ |
| name, \ |
| iree_vm_ImportFunctionDef_vec_len(verify_state->imported_functions)); \ |
| } |
| #define VM_VerifyFunctionOrdinal(name) \ |
| if (IREE_UNLIKELY((name)) >= iree_vm_FunctionDescriptor_vec_len( \ |
| verify_state->function_descriptors)) { \ |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ |
| "function ordinal %u out of range %zu", (name), \ |
| iree_vm_FunctionDescriptor_vec_len( \ |
| verify_state->function_descriptors)); \ |
| } |
| #define VM_VerifyGlobalAttr(name) VM_VerifyConstI32(name) |
| #define VM_VerifyRwdataOffset(name, access_length) \ |
| if (IREE_UNLIKELY(((name) + (access_length)) > \ |
| verify_state->rwdata_storage_size)) { \ |
| return iree_make_status( \ |
| IREE_STATUS_OUT_OF_RANGE, \ |
| "global byte_offset out of range: %d (rwdata=%" PRIhsz ")", (name), \ |
| verify_state->rwdata_storage_size); \ |
| } |
| #define VM_VerifyGlobalRefOrdinal(name) \ |
| if (IREE_UNLIKELY((name) >= verify_state->global_ref_count)) { \ |
| return iree_make_status( \ |
| IREE_STATUS_OUT_OF_RANGE, \ |
| "global ref ordinal out of range: %d (table=%" PRIhsz ")", (name), \ |
| verify_state->global_ref_count); \ |
| } |
| #define VM_VerifyRodataAttr(name) VM_VerifyConstI32(name) |
| #define VM_VerifyRodataOrdinal(name) \ |
| if (IREE_UNLIKELY((name) >= verify_state->rodata_ref_count)) { \ |
| return iree_make_status( \ |
| IREE_STATUS_OUT_OF_RANGE, \ |
| "rodata ref ordinal out of range: %d (table=%" PRIhsz ")", (name), \ |
| verify_state->rodata_ref_count); \ |
| } |
| #define VM_VerifyType(name) \ |
| IREE_VM_VERIFY_PC_RANGE(pc + 4, max_pc); \ |
| uint32_t name##_id = OP_I32(0); \ |
| if (IREE_UNLIKELY(name##_id >= module->type_count)) { \ |
| return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ |
| "type id ordinal out of range: %d (table=%" PRIhsz \ |
| ")", \ |
| name##_id, module->type_count); \ |
| } \ |
| const iree_vm_type_def_t* name = &module->type_table[name##_id]; \ |
| (void)(name); \ |
| pc += 4; |
| #define VM_VerifyTypeOf(name) VM_VerifyType(name) |
| #define VM_VerifyIntAttr32(name) VM_VerifyConstI32(name) |
| #define VM_VerifyIntAttr64(name) VM_VerifyConstI64(name) |
| #define VM_VerifyFloatAttr32(name) VM_VerifyConstF32(name) |
| #define VM_VerifyFloatAttr64(name) VM_VerifyConstF64(name) |
| #define VM_VerifyStrAttr(name, out_str) \ |
| IREE_VM_VERIFY_PC_RANGE(pc + 2, max_pc); \ |
| (out_str)->size = (iree_host_size_t)OP_I16(0); \ |
| IREE_VM_VERIFY_PC_RANGE(pc + 2 + (out_str)->size, max_pc); \ |
| (out_str)->data = (const char*)&bytecode_data[pc + 2]; \ |
| pc += 2 + (out_str)->size; |
| |
| #define VM_VerifyBranchTarget(name) \ |
| VM_VerifyConstI32(name##_pc); \ |
| iree_vm_bytecode_block_t* name = NULL; \ |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_block_list_insert( \ |
| &verify_state->block_list, name##_pc, &name)); |
| #define VM_VerifyBranchOperands(name) \ |
| VM_AlignPC(pc, IREE_REGISTER_ORDINAL_SIZE); \ |
| IREE_VM_VERIFY_PC_RANGE(pc + IREE_REGISTER_ORDINAL_SIZE, max_pc); \ |
| const iree_vm_register_remap_list_t* name = \ |
| (const iree_vm_register_remap_list_t*)&bytecode_data[pc]; \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; \ |
| IREE_VM_VERIFY_PC_RANGE(pc + (name)->size * 2 * IREE_REGISTER_ORDINAL_SIZE, \ |
| max_pc); \ |
| pc += (name)->size * 2 * IREE_REGISTER_ORDINAL_SIZE; \ |
| for (uint16_t i = 0; i < name->size; ++i) { \ |
| IREE_VM_VERIFY_REG_ANY(name->pairs[i].src_reg); \ |
| IREE_VM_VERIFY_REG_ANY(name->pairs[i].dst_reg); \ |
| } |
| |
| #define VM_VerifyOperandRegI32(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_I32(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyOperandRegI64(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_I64(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyOperandRegI64HostSize(name) VM_VerifyOperandRegI64(name) |
| #define VM_VerifyOperandRegF32(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_F32(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyOperandRegF64(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_F32(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyOperandRegRef(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_REF(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyVariadicOperands(name) \ |
| VM_AlignPC(pc, IREE_REGISTER_ORDINAL_SIZE); \ |
| IREE_VM_VERIFY_PC_RANGE(pc + IREE_REGISTER_ORDINAL_SIZE, max_pc); \ |
| const iree_vm_register_list_t* name = \ |
| (const iree_vm_register_list_t*)&bytecode_data[pc]; \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; \ |
| IREE_VM_VERIFY_PC_RANGE(pc + (name)->size * IREE_REGISTER_ORDINAL_SIZE, \ |
| max_pc); \ |
| pc += (name)->size * IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyVariadicOperandsI32(name) \ |
| VM_VerifyVariadicOperands(name); \ |
| for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ |
| IREE_VM_VERIFY_REG_I32((name)->registers[__i]); \ |
| } |
| #define VM_VerifyVariadicOperandsI64(name) \ |
| VM_VerifyVariadicOperands(name); \ |
| for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ |
| IREE_VM_VERIFY_REG_I64((name)->registers[__i]); \ |
| } |
| #define VM_VerifyVariadicOperandsF32(name) \ |
| VM_VerifyVariadicOperands(name); \ |
| for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ |
| IREE_VM_VERIFY_REG_F32((name)->registers[__i]); \ |
| } |
| #define VM_VerifyVariadicOperandsF64(name) \ |
| VM_VerifyVariadicOperands(name); \ |
| for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ |
| IREE_VM_VERIFY_REG_F64((name)->registers[__i]); \ |
| } |
| #define VM_VerifyVariadicOperandsRef(name, type_def) \ |
| VM_VerifyVariadicOperands(name); \ |
| for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ |
| IREE_VM_VERIFY_REG_REF((name)->registers[__i]); \ |
| } |
| #define VM_VerifyVariadicOperandsAny(name) \ |
| VM_VerifyVariadicOperands(name); \ |
| for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ |
| IREE_VM_VERIFY_REG_ANY((name)->registers[__i]); \ |
| } |
| #define VM_VerifyResultRegI32(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_I32(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyResultRegI64(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_I64(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyResultRegF32(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_F32(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyResultRegF64(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_F64(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyResultRegRef(name) \ |
| IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ |
| IREE_VM_VERIFY_REG_REF(name##_ordinal); \ |
| pc += IREE_REGISTER_ORDINAL_SIZE; |
| #define VM_VerifyVariadicResultsAny(name) VM_VerifyVariadicOperandsAny(name) |
| |
| #define VERIFY_OP_CORE_UNARY_I32(op_name) \ |
| VERIFY_OP(CORE, op_name, { \ |
| VM_VerifyOperandRegI32(operand); \ |
| VM_VerifyResultRegI32(result); \ |
| }); |
| |
| #define VERIFY_OP_CORE_UNARY_I64(op_name) \ |
| VERIFY_OP(CORE, op_name, { \ |
| VM_VerifyOperandRegI64(operand); \ |
| VM_VerifyResultRegI64(result); \ |
| }); |
| |
| #define VERIFY_OP_CORE_BINARY_I32(op_name) \ |
| VERIFY_OP(CORE, op_name, { \ |
| VM_VerifyOperandRegI32(lhs); \ |
| VM_VerifyOperandRegI32(rhs); \ |
| VM_VerifyResultRegI32(result); \ |
| }); |
| |
| #define VERIFY_OP_CORE_BINARY_I64(op_name) \ |
| VERIFY_OP(CORE, op_name, { \ |
| VM_VerifyOperandRegI64(lhs); \ |
| VM_VerifyOperandRegI64(rhs); \ |
| VM_VerifyResultRegI64(result); \ |
| }); |
| |
| #define VERIFY_OP_CORE_TERNARY_I32(op_name) \ |
| VERIFY_OP(CORE, op_name, { \ |
| VM_VerifyOperandRegI32(a); \ |
| VM_VerifyOperandRegI32(b); \ |
| VM_VerifyOperandRegI32(c); \ |
| VM_VerifyResultRegI32(result); \ |
| }); |
| |
| #define VERIFY_OP_CORE_TERNARY_I64(op_name) \ |
| VERIFY_OP(CORE, op_name, { \ |
| VM_VerifyOperandRegI64(a); \ |
| VM_VerifyOperandRegI64(b); \ |
| VM_VerifyOperandRegI64(c); \ |
| VM_VerifyResultRegI64(result); \ |
| }); |
| |
| #define VERIFY_OP_EXT_F32_UNARY_F32(op_name) \ |
| VERIFY_OP(EXT_F32, op_name, { \ |
| VM_VerifyOperandRegF32(operand); \ |
| VM_VerifyResultRegF32(result); \ |
| }); |
| |
| #define VERIFY_OP_EXT_F32_BINARY_F32(op_name) \ |
| VERIFY_OP(EXT_F32, op_name, { \ |
| VM_VerifyOperandRegF32(lhs); \ |
| VM_VerifyOperandRegF32(rhs); \ |
| VM_VerifyResultRegF32(result); \ |
| }); |
| |
| #define VERIFY_OP_EXT_F32_TERNARY_F32(op_name) \ |
| VERIFY_OP(EXT_F32, op_name, { \ |
| VM_VerifyOperandRegF32(a); \ |
| VM_VerifyOperandRegF32(b); \ |
| VM_VerifyOperandRegF32(c); \ |
| VM_VerifyResultRegF32(result); \ |
| }); |
| |
| #define VERIFY_OP_EXT_F64_UNARY_F64(op_name) \ |
| VERIFY_OP(EXT_F64, op_name, { \ |
| VM_VerifyOperandRegF64(operand); \ |
| VM_VerifyResultRegF64(result); \ |
| }); |
| |
| #define VERIFY_OP_EXT_F64_BINARY_F64(op_name) \ |
| VERIFY_OP(EXT_F64, op_name, { \ |
| VM_VerifyOperandRegF64(lhs); \ |
| VM_VerifyOperandRegF64(rhs); \ |
| VM_VerifyResultRegF64(result); \ |
| }); |
| |
| #define VERIFY_OP_EXT_F64_TERNARY_F64(op_name) \ |
| VERIFY_OP(EXT_F64, op_name, { \ |
| VM_VerifyOperandRegF64(a); \ |
| VM_VerifyOperandRegF64(b); \ |
| VM_VerifyOperandRegF64(c); \ |
| VM_VerifyResultRegF64(result); \ |
| }); |
| |
| //===----------------------------------------------------------------------===// |
| // Call verification |
| //===----------------------------------------------------------------------===// |
| |
| static iree_status_t iree_vm_bytecode_function_get_cconv_fragments( |
| iree_vm_FunctionSignatureDef_table_t signature_def, |
| iree_string_view_t* out_arguments, iree_string_view_t* out_results) { |
| flatbuffers_string_t cconv_str = |
| iree_vm_FunctionSignatureDef_calling_convention(signature_def); |
| iree_vm_function_signature_t signature = { |
| .calling_convention = |
| iree_make_string_view(cconv_str, flatbuffers_string_len(cconv_str)), |
| }; |
| return iree_vm_function_call_get_cconv_fragments(&signature, out_arguments, |
| out_results); |
| } |
| |
| static iree_status_t iree_vm_bytecode_function_count_cconv_regs( |
| iree_string_view_t cconv_fragment, iree_host_size_t* out_i32_count, |
| iree_host_size_t* out_ref_count) { |
| iree_host_size_t i32_count = 0; |
| iree_host_size_t ref_count = 0; |
| for (iree_host_size_t i = 0; i < cconv_fragment.size; ++i) { |
| switch (cconv_fragment.data[i]) { |
| default: |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "unsupported cconv fragment char '%c'", |
| cconv_fragment.data[i]); |
| case IREE_VM_CCONV_TYPE_VOID: |
| break; |
| case IREE_VM_CCONV_TYPE_I32: |
| case IREE_VM_CCONV_TYPE_F32: |
| ++i32_count; |
| break; |
| case IREE_VM_CCONV_TYPE_I64: |
| case IREE_VM_CCONV_TYPE_F64: |
| if ((i32_count % 2) != 0) { |
| // Unaligned; pad an i32 register to get to i64 alignment. |
| ++i32_count; |
| } |
| i32_count += 2; |
| break; |
| case IREE_VM_CCONV_TYPE_REF: |
| ++ref_count; |
| break; |
| case IREE_VM_CCONV_TYPE_SPAN_START: |
| return iree_make_status( |
| IREE_STATUS_INVALID_ARGUMENT, |
| "internal functions cannot accept variadic arguments"); |
| } |
| } |
| *out_i32_count = i32_count; |
| *out_ref_count = ref_count; |
| return iree_ok_status(); |
| } |
| |
| static iree_status_t iree_vm_bytecode_function_verify_arguments( |
| const iree_vm_bytecode_verify_state_t* verify_state) { |
| iree_host_size_t args_i32 = 0; |
| iree_host_size_t args_ref = 0; |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_count_cconv_regs( |
| verify_state->cconv_arguments, &args_i32, &args_ref)); |
| if (verify_state->i32_register_count < args_i32 || |
| verify_state->ref_register_count < args_ref) { |
| return iree_make_status( |
| IREE_STATUS_INVALID_ARGUMENT, |
| "insufficient register storage for function arguments/results"); |
| } |
| return iree_ok_status(); |
| } |
| |
| static iree_status_t iree_vm_bytecode_function_verify_cconv_register( |
| const iree_vm_bytecode_verify_state_t* verify_state, char cconv_type, |
| const iree_vm_register_list_t* IREE_RESTRICT reg_list, int reg_i) { |
| if (reg_i >= reg_list->size) { |
| return iree_make_status( |
| IREE_STATUS_OUT_OF_RANGE, |
| "register list underflow (have %u, trying to access %u)", |
| reg_list->size, reg_i); |
| } |
| switch (cconv_type) { |
| case IREE_VM_CCONV_TYPE_I32: |
| case IREE_VM_CCONV_TYPE_F32: { |
| IREE_VM_VERIFY_REG_ORDINAL_X32(reg_list->registers[reg_i], "i32/f32"); |
| } break; |
| case IREE_VM_CCONV_TYPE_I64: |
| case IREE_VM_CCONV_TYPE_F64: { |
| IREE_VM_VERIFY_REG_ORDINAL_X64(reg_list->registers[reg_i], "i64/f64"); |
| } break; |
| case IREE_VM_CCONV_TYPE_REF: { |
| IREE_VM_VERIFY_REG_REF(reg_list->registers[reg_i]); |
| } break; |
| default: |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "unsupported cconv fragment char '%c'", |
| cconv_type); |
| } |
| return iree_ok_status(); |
| } |
| |
| static iree_status_t iree_vm_bytecode_function_verify_cconv_registers( |
| const iree_vm_bytecode_verify_state_t* verify_state, |
| iree_string_view_t cconv_fragment, |
| const iree_vm_register_list_t* IREE_RESTRICT segment_size_list, |
| const iree_vm_register_list_t* IREE_RESTRICT reg_list) { |
| for (uint16_t i = 0, seg_i = 0, reg_i = 0; i < cconv_fragment.size; |
| ++i, ++seg_i) { |
| switch (cconv_fragment.data[i]) { |
| case IREE_VM_CCONV_TYPE_VOID: |
| break; |
| case IREE_VM_CCONV_TYPE_I32: |
| case IREE_VM_CCONV_TYPE_F32: |
| case IREE_VM_CCONV_TYPE_I64: |
| case IREE_VM_CCONV_TYPE_F64: |
| case IREE_VM_CCONV_TYPE_REF: { |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_verify_cconv_register( |
| verify_state, cconv_fragment.data[i], reg_list, reg_i++)); |
| } break; |
| case IREE_VM_CCONV_TYPE_SPAN_START: { |
| if (!segment_size_list) { |
| return iree_make_status( |
| IREE_STATUS_INVALID_ARGUMENT, |
| "function is variadic but no segment size list provided"); |
| } else if (seg_i >= segment_size_list->size) { |
| return iree_make_status( |
| IREE_STATUS_OUT_OF_RANGE, |
| "segment size list underflow (have %u, trying to access %u)", |
| segment_size_list->size, seg_i); |
| } |
| uint16_t span_count = segment_size_list->registers[seg_i]; |
| if (!span_count) { |
| // No items; skip the span. |
| do { |
| ++i; |
| } while (i < cconv_fragment.size && |
| cconv_fragment.data[i] != IREE_VM_CCONV_TYPE_SPAN_END); |
| continue; |
| } |
| uint16_t span_start_i = i + 1; |
| for (uint16_t j = 0; j < span_count; ++j) { |
| for (i = span_start_i; |
| i < cconv_fragment.size && |
| cconv_fragment.data[i] != IREE_VM_CCONV_TYPE_SPAN_END; |
| ++i) { |
| // TODO(benvanik): share with switch above. |
| switch (cconv_fragment.data[i]) { |
| case IREE_VM_CCONV_TYPE_VOID: |
| break; |
| case IREE_VM_CCONV_TYPE_I32: |
| case IREE_VM_CCONV_TYPE_F32: |
| case IREE_VM_CCONV_TYPE_I64: |
| case IREE_VM_CCONV_TYPE_F64: |
| case IREE_VM_CCONV_TYPE_REF: { |
| IREE_RETURN_IF_ERROR( |
| iree_vm_bytecode_function_verify_cconv_register( |
| verify_state, cconv_fragment.data[i], reg_list, |
| reg_i++)); |
| } break; |
| default: |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "unsupported cconv fragment char '%c'", |
| cconv_fragment.data[i]); |
| } |
| } |
| } |
| } break; |
| default: |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "unsupported cconv fragment char '%c'", |
| cconv_fragment.data[i]); |
| } |
| } |
| return iree_ok_status(); |
| } |
| |
| static iree_status_t iree_vm_bytecode_function_verify_call( |
| const iree_vm_bytecode_verify_state_t* verify_state, |
| iree_vm_FunctionSignatureDef_table_t signature_def, |
| const iree_vm_register_list_t* IREE_RESTRICT segment_size_list, |
| const iree_vm_register_list_t* IREE_RESTRICT src_reg_list, |
| const iree_vm_register_list_t* IREE_RESTRICT dst_reg_list) { |
| iree_string_view_t arguments = iree_string_view_empty(); |
| iree_string_view_t results = iree_string_view_empty(); |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_get_cconv_fragments( |
| signature_def, &arguments, &results)); |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_verify_cconv_registers( |
| verify_state, arguments, segment_size_list, src_reg_list)); |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_verify_cconv_registers( |
| verify_state, results, /*segment_sizes=*/NULL, dst_reg_list)); |
| return iree_ok_status(); |
| } |
| |
| //===----------------------------------------------------------------------===// |
| // Bytecode verification |
| //===----------------------------------------------------------------------===// |
| |
| #define VERIFY_OP(ext, op_name, body) \ |
| case IREE_VM_OP_##ext##_##op_name: { \ |
| body; \ |
| } break; |
| |
| #define BEGIN_VERIFY_PREFIX(op_name, ext) \ |
| case IREE_VM_OP_CORE_##op_name: { \ |
| IREE_VM_VERIFY_PC_RANGE(pc + 1, max_pc); \ |
| IREE_VM_VERIFY_REQUIREMENT(ext); \ |
| switch (bytecode_data[pc++]) { |
| #define END_VERIFY_PREFIX() \ |
| default: \ |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ |
| "unhandled ext opcode"); \ |
| } \ |
| break; \ |
| } |
| #define UNHANDLED_VERIFY_PREFIX(op_name, ext) \ |
| case IREE_VM_OP_CORE_##op_name: { \ |
| return iree_vm_check_feature_mismatch(__FILE__, __LINE__, ext, \ |
| function_descriptor->requirements); \ |
| } |
| |
| static iree_status_t iree_vm_bytecode_function_verify_bytecode_op( |
| iree_vm_bytecode_module_t* module, |
| iree_vm_bytecode_verify_state_t* verify_state, |
| iree_vm_FunctionSignatureDef_table_t function_signature, |
| iree_vm_FunctionDescriptor_struct_t function_descriptor, |
| iree_const_byte_span_t function_bytecode, uint32_t start_pc, |
| uint32_t max_pc, uint32_t* out_next_pc) { |
| *out_next_pc = 0; |
| uint32_t pc = start_pc; |
| const uint8_t* bytecode_data = function_bytecode.data; |
| |
| // NOTE: we keep this as simple as possible so that we can one day auto |
| // generate it from tblgen, which has all the encodings in a similar form |
| // such that we could do string substitution to get the verifier macros. |
| |
| // All ops except for Block must be inside of a block. We hoist that check |
| // that here before switching out. |
| IREE_VM_VERIFY_PC_RANGE(pc + 1, max_pc); |
| if (verify_state->in_block == 0) { |
| // If not in a block then the next opcode must be a block. |
| if (bytecode_data[pc] != IREE_VM_OP_CORE_Block) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "op at pc %08X is not in a block", pc); |
| } |
| } else { |
| // If in a block then the next opcode must not be a block. |
| if (bytecode_data[pc] == IREE_VM_OP_CORE_Block) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "op at pc %08X is a block while still in a block", |
| pc); |
| } |
| } |
| |
| // Get primary opcode. All ops have at least 1 byte. |
| switch (bytecode_data[pc++]) { |
| //===------------------------------------------------------------------===// |
| // Globals |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP(CORE, GlobalLoadI32, { |
| VM_VerifyGlobalAttr(byte_offset); |
| VM_VerifyRwdataOffset(byte_offset, 4); |
| VM_VerifyResultRegI32(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalStoreI32, { |
| VM_VerifyGlobalAttr(byte_offset); |
| VM_VerifyRwdataOffset(byte_offset, 4); |
| VM_VerifyOperandRegI32(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalLoadIndirectI32, { |
| VM_VerifyOperandRegI32(byte_offset); |
| // NOTE: we have to verify the offset at runtime. |
| VM_VerifyResultRegI32(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalStoreIndirectI32, { |
| VM_VerifyOperandRegI32(byte_offset); |
| // NOTE: we have to verify the offset at runtime. |
| VM_VerifyOperandRegI32(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalLoadI64, { |
| VM_VerifyGlobalAttr(byte_offset); |
| VM_VerifyRwdataOffset(byte_offset, 8); |
| VM_VerifyResultRegI64(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalStoreI64, { |
| VM_VerifyGlobalAttr(byte_offset); |
| VM_VerifyRwdataOffset(byte_offset, 8); |
| VM_VerifyOperandRegI64(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalLoadIndirectI64, { |
| VM_VerifyOperandRegI32(byte_offset); |
| // NOTE: we have to verify the offset at runtime. |
| VM_VerifyResultRegI64(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalStoreIndirectI64, { |
| VM_VerifyOperandRegI32(byte_offset); |
| // NOTE: we have to verify the offset at runtime. |
| VM_VerifyOperandRegI64(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalLoadRef, { |
| VM_VerifyGlobalAttr(global); |
| VM_VerifyGlobalRefOrdinal(global); |
| VM_VerifyTypeOf(type_def); |
| VM_VerifyResultRegRef(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalStoreRef, { |
| VM_VerifyGlobalAttr(global); |
| VM_VerifyGlobalRefOrdinal(global); |
| VM_VerifyTypeOf(type_def); |
| VM_VerifyOperandRegRef(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalLoadIndirectRef, { |
| VM_VerifyOperandRegI32(global); |
| // NOTE: we have to verify the ordinal at runtime. |
| VM_VerifyTypeOf(type_def); |
| VM_VerifyResultRegRef(value); |
| }); |
| |
| VERIFY_OP(CORE, GlobalStoreIndirectRef, { |
| VM_VerifyOperandRegI32(global); |
| // NOTE: we have to verify the ordinal at runtime. |
| VM_VerifyTypeOf(type_def); |
| VM_VerifyOperandRegRef(value); |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Constants |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP(CORE, ConstI32, { |
| VM_VerifyIntAttr32(value); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| VERIFY_OP(CORE, ConstI32Zero, { VM_VerifyResultRegI32(result); }); |
| |
| VERIFY_OP(CORE, ConstI64, { |
| VM_VerifyIntAttr64(value); |
| VM_VerifyResultRegI64(result); |
| }); |
| |
| VERIFY_OP(CORE, ConstI64Zero, { VM_VerifyResultRegI64(result); }); |
| |
| VERIFY_OP(CORE, ConstRefZero, { VM_VerifyResultRegRef(result); }); |
| |
| VERIFY_OP(CORE, ConstRefRodata, { |
| VM_VerifyRodataAttr(rodata); |
| VM_VerifyRodataOrdinal(rodata); |
| VM_VerifyResultRegRef(value); |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Buffers |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP(CORE, BufferAlloc, { |
| VM_VerifyOperandRegI64HostSize(length); |
| VM_VerifyOperandRegI32(alignment); |
| VM_VerifyResultRegRef(result); |
| }); |
| |
| VERIFY_OP(CORE, BufferClone, { |
| VM_VerifyOperandRegRef(source); |
| VM_VerifyOperandRegI64HostSize(offset); |
| VM_VerifyOperandRegI64HostSize(length); |
| VM_VerifyOperandRegI32(alignment); |
| VM_VerifyResultRegRef(result); |
| }); |
| |
| VERIFY_OP(CORE, BufferLength, { |
| VM_VerifyOperandRegRef(buffer); |
| VM_VerifyResultRegI64(result); |
| }); |
| |
| VERIFY_OP(CORE, BufferCopy, { |
| VM_VerifyOperandRegRef(source_buffer); |
| VM_VerifyOperandRegI64HostSize(source_offset); |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI64HostSize(length); |
| }); |
| |
| VERIFY_OP(CORE, BufferCompare, { |
| VM_VerifyOperandRegRef(lhs_buffer); |
| VM_VerifyOperandRegI64HostSize(lhs_offset); |
| VM_VerifyOperandRegRef(rhs_buffer); |
| VM_VerifyOperandRegI64HostSize(rhs_offset); |
| VM_VerifyOperandRegI64HostSize(length); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| VERIFY_OP(CORE, BufferFillI8, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI64HostSize(length); |
| VM_VerifyOperandRegI32(value); |
| }); |
| VERIFY_OP(CORE, BufferFillI16, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI64HostSize(length); |
| VM_VerifyOperandRegI32(value); |
| }); |
| VERIFY_OP(CORE, BufferFillI32, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI64HostSize(length); |
| VM_VerifyOperandRegI32(value); |
| }); |
| VERIFY_OP(CORE, BufferFillI64, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI64HostSize(length); |
| VM_VerifyOperandRegI64(value); |
| }); |
| |
| VERIFY_OP(CORE, BufferLoadI8U, { |
| VM_VerifyOperandRegRef(source_buffer); |
| VM_VerifyOperandRegI64HostSize(source_offset); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(CORE, BufferLoadI8S, { |
| VM_VerifyOperandRegRef(source_buffer); |
| VM_VerifyOperandRegI64HostSize(source_offset); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(CORE, BufferLoadI16U, { |
| VM_VerifyOperandRegRef(source_buffer); |
| VM_VerifyOperandRegI64HostSize(source_offset); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(CORE, BufferLoadI16S, { |
| VM_VerifyOperandRegRef(source_buffer); |
| VM_VerifyOperandRegI64HostSize(source_offset); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(CORE, BufferLoadI32, { |
| VM_VerifyOperandRegRef(source_buffer); |
| VM_VerifyOperandRegI64HostSize(source_offset); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(CORE, BufferLoadI64, { |
| VM_VerifyOperandRegRef(source_buffer); |
| VM_VerifyOperandRegI64HostSize(source_offset); |
| VM_VerifyResultRegI64(result); |
| }); |
| |
| VERIFY_OP(CORE, BufferStoreI8, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI32(value); |
| }); |
| VERIFY_OP(CORE, BufferStoreI16, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI32(value); |
| }); |
| VERIFY_OP(CORE, BufferStoreI32, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI32(value); |
| }); |
| VERIFY_OP(CORE, BufferStoreI64, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI64(value); |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Lists |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP(CORE, ListAlloc, { |
| VM_VerifyTypeOf(element_type); |
| VM_VerifyOperandRegI32(initial_capacity); |
| VM_VerifyResultRegRef(result); |
| }); |
| |
| VERIFY_OP(CORE, ListReserve, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(minimum_capacity); |
| }); |
| |
| VERIFY_OP(CORE, ListSize, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| VERIFY_OP(CORE, ListResize, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(new_size); |
| }); |
| |
| VERIFY_OP(CORE, ListGetI32, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| VERIFY_OP(CORE, ListSetI32, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyOperandRegI32(raw_value); |
| }); |
| |
| VERIFY_OP(CORE, ListGetI64, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyResultRegI64(result); |
| }); |
| |
| VERIFY_OP(CORE, ListSetI64, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyOperandRegI64(value); |
| }); |
| |
| VERIFY_OP(CORE, ListGetRef, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyTypeOf(type_def); |
| VM_VerifyResultRegRef(result); |
| }); |
| |
| VERIFY_OP(CORE, ListSetRef, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyOperandRegRef(value); |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Conditional assignment |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP(CORE, SelectI32, { |
| VM_VerifyOperandRegI32(condition); |
| VM_VerifyOperandRegI32(true_value); |
| VM_VerifyOperandRegI32(false_value); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| VERIFY_OP(CORE, SelectI64, { |
| VM_VerifyOperandRegI32(condition); |
| VM_VerifyOperandRegI64(true_value); |
| VM_VerifyOperandRegI64(false_value); |
| VM_VerifyResultRegI64(result); |
| }); |
| |
| VERIFY_OP(CORE, SelectRef, { |
| VM_VerifyOperandRegI32(condition); |
| // TODO(benvanik): remove the type_id and use either LHS/RHS (if both are |
| // null then output is always null so no need to know the type). |
| VM_VerifyTypeOf(true_value_type_def); |
| VM_VerifyOperandRegRef(true_value); |
| VM_VerifyOperandRegRef(false_value); |
| VM_VerifyResultRegRef(result); |
| }); |
| |
| VERIFY_OP(CORE, SwitchI32, { |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyOperandRegI32(default_value); |
| VM_VerifyVariadicOperandsI32(values); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| VERIFY_OP(CORE, SwitchI64, { |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyOperandRegI64(default_value); |
| VM_VerifyVariadicOperandsI64(values); |
| VM_VerifyResultRegI64(result); |
| }); |
| |
| VERIFY_OP(CORE, SwitchRef, { |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyTypeOf(type_def); |
| VM_VerifyOperandRegRef(default_value); |
| VM_VerifyVariadicOperandsRef(values, type_def); |
| VM_VerifyResultRegRef(result); |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Native integer arithmetic |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP_CORE_BINARY_I32(AddI32); |
| VERIFY_OP_CORE_BINARY_I32(SubI32); |
| VERIFY_OP_CORE_BINARY_I32(MulI32); |
| VERIFY_OP_CORE_BINARY_I32(DivI32S); |
| VERIFY_OP_CORE_BINARY_I32(DivI32U); |
| VERIFY_OP_CORE_BINARY_I32(RemI32S); |
| VERIFY_OP_CORE_BINARY_I32(RemI32U); |
| VERIFY_OP_CORE_TERNARY_I32(FMAI32); |
| VERIFY_OP_CORE_UNARY_I32(AbsI32); |
| VERIFY_OP_CORE_BINARY_I32(MinI32S); |
| VERIFY_OP_CORE_BINARY_I32(MinI32U); |
| VERIFY_OP_CORE_BINARY_I32(MaxI32S); |
| VERIFY_OP_CORE_BINARY_I32(MaxI32U); |
| VERIFY_OP_CORE_UNARY_I32(NotI32); |
| VERIFY_OP_CORE_BINARY_I32(AndI32); |
| VERIFY_OP_CORE_BINARY_I32(OrI32); |
| VERIFY_OP_CORE_BINARY_I32(XorI32); |
| VERIFY_OP_CORE_UNARY_I32(CtlzI32); |
| |
| VERIFY_OP_CORE_BINARY_I64(AddI64); |
| VERIFY_OP_CORE_BINARY_I64(SubI64); |
| VERIFY_OP_CORE_BINARY_I64(MulI64); |
| VERIFY_OP_CORE_BINARY_I64(DivI64S); |
| VERIFY_OP_CORE_BINARY_I64(DivI64U); |
| VERIFY_OP_CORE_BINARY_I64(RemI64S); |
| VERIFY_OP_CORE_BINARY_I64(RemI64U); |
| VERIFY_OP_CORE_TERNARY_I64(FMAI64); |
| VERIFY_OP_CORE_UNARY_I64(AbsI64); |
| VERIFY_OP_CORE_BINARY_I64(MinI64S); |
| VERIFY_OP_CORE_BINARY_I64(MinI64U); |
| VERIFY_OP_CORE_BINARY_I64(MaxI64S); |
| VERIFY_OP_CORE_BINARY_I64(MaxI64U); |
| VERIFY_OP_CORE_UNARY_I64(NotI64); |
| VERIFY_OP_CORE_BINARY_I64(AndI64); |
| VERIFY_OP_CORE_BINARY_I64(OrI64); |
| VERIFY_OP_CORE_BINARY_I64(XorI64); |
| VERIFY_OP_CORE_UNARY_I64(CtlzI64); |
| |
| //===------------------------------------------------------------------===// |
| // Casting and type conversion/emulation |
| //===------------------------------------------------------------------===// |
| |
| // NOTE: these all operate on 32-bit registers. |
| VERIFY_OP_CORE_UNARY_I32(TruncI32I8); |
| VERIFY_OP_CORE_UNARY_I32(TruncI32I16); |
| VERIFY_OP_CORE_UNARY_I32(ExtI8I32S); |
| VERIFY_OP_CORE_UNARY_I32(ExtI8I32U); |
| VERIFY_OP_CORE_UNARY_I32(ExtI16I32S); |
| VERIFY_OP_CORE_UNARY_I32(ExtI16I32U); |
| |
| // NOTE: 64-bit ones are actually changing register widths. |
| VERIFY_OP(CORE, TruncI64I32, { |
| VM_VerifyOperandRegI64(operand); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(CORE, ExtI32I64S, { |
| VM_VerifyOperandRegI32(operand); |
| VM_VerifyResultRegI64(result); |
| }); |
| VERIFY_OP(CORE, ExtI32I64U, { |
| VM_VerifyOperandRegI32(operand); |
| VM_VerifyResultRegI64(result); |
| }); |
| |
| VERIFY_OP(CORE, CastAnyRef, { |
| VM_VerifyOperandRegRef(operand); |
| VM_VerifyTypeOf(result); |
| VM_VerifyResultRegRef(result); |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Native bitwise shifts and rotates |
| //===------------------------------------------------------------------===// |
| |
| #define VERIFY_OP_CORE_SHIFT_I32(op_name) \ |
| VERIFY_OP(CORE, op_name, { \ |
| VM_VerifyOperandRegI32(operand); \ |
| VM_VerifyOperandRegI32(amount); \ |
| VM_VerifyResultRegI32(result); \ |
| }); |
| |
| VERIFY_OP_CORE_SHIFT_I32(ShlI32); |
| VERIFY_OP_CORE_SHIFT_I32(ShrI32S); |
| VERIFY_OP_CORE_SHIFT_I32(ShrI32U); |
| |
| #define VERIFY_OP_CORE_SHIFT_I64(op_name) \ |
| VERIFY_OP(CORE, op_name, { \ |
| VM_VerifyOperandRegI64(operand); \ |
| VM_VerifyOperandRegI32(amount); \ |
| VM_VerifyResultRegI64(result); \ |
| }); |
| |
| VERIFY_OP_CORE_SHIFT_I64(ShlI64); |
| VERIFY_OP_CORE_SHIFT_I64(ShrI64S); |
| VERIFY_OP_CORE_SHIFT_I64(ShrI64U); |
| |
| //===------------------------------------------------------------------===// |
| // Comparison ops |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP_CORE_BINARY_I32(CmpEQI32); |
| VERIFY_OP_CORE_BINARY_I32(CmpNEI32); |
| VERIFY_OP_CORE_BINARY_I32(CmpLTI32S); |
| VERIFY_OP_CORE_BINARY_I32(CmpLTI32U); |
| VERIFY_OP_CORE_UNARY_I32(CmpNZI32); |
| |
| #define VERIFY_OP_CORE_CMP_I64(op_name) \ |
| VERIFY_OP(CORE, op_name, { \ |
| VM_VerifyOperandRegI64(lhs); \ |
| VM_VerifyOperandRegI64(rhs); \ |
| VM_VerifyResultRegI32(result); \ |
| }); |
| |
| VERIFY_OP_CORE_CMP_I64(CmpEQI64); |
| VERIFY_OP_CORE_CMP_I64(CmpNEI64); |
| VERIFY_OP_CORE_CMP_I64(CmpLTI64S); |
| VERIFY_OP_CORE_CMP_I64(CmpLTI64U); |
| VERIFY_OP(CORE, CmpNZI64, { |
| VM_VerifyOperandRegI64(operand); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| VERIFY_OP(CORE, CmpEQRef, { |
| VM_VerifyOperandRegRef(lhs); |
| VM_VerifyOperandRegRef(rhs); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(CORE, CmpNERef, { |
| VM_VerifyOperandRegRef(lhs); |
| VM_VerifyOperandRegRef(rhs); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(CORE, CmpNZRef, { |
| VM_VerifyOperandRegRef(operand); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Control flow |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP(CORE, Block, { |
| // Define the new block in the block list. It may already be declared from |
| // a prior branch. |
| iree_vm_bytecode_block_t* block = NULL; |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_block_list_insert( |
| &verify_state->block_list, pc - 1, &block)); |
| block->defined = 1; |
| verify_state->in_block = 1; |
| }); |
| |
| VERIFY_OP(CORE, Branch, { |
| VM_VerifyBranchTarget(dest_pc); |
| VM_VerifyBranchOperands(operands); |
| verify_state->in_block = 0; // terminator |
| }); |
| |
| VERIFY_OP(CORE, CondBranch, { |
| VM_VerifyOperandRegI32(condition); |
| VM_VerifyBranchTarget(true_dest_pc); |
| VM_VerifyBranchOperands(true_operands); |
| VM_VerifyBranchTarget(false_dest_pc); |
| VM_VerifyBranchOperands(false_operands); |
| verify_state->in_block = 0; // terminator |
| }); |
| |
| VERIFY_OP(CORE, Call, { |
| VM_VerifyFuncAttr(callee_ordinal); |
| VM_VerifyVariadicOperandsAny(operands); |
| VM_VerifyVariadicResultsAny(results); |
| if (VM_IsImportOrdinal(callee_ordinal)) { |
| VM_UnmaskImportOrdinal(callee_ordinal); |
| VM_VerifyImportOrdinal(callee_ordinal); |
| iree_vm_ImportFunctionDef_table_t import_def = |
| iree_vm_ImportFunctionDef_vec_at(verify_state->imported_functions, |
| callee_ordinal); |
| IREE_RETURN_IF_ERROR( |
| iree_vm_bytecode_function_verify_call( |
| verify_state, iree_vm_ImportFunctionDef_signature(import_def), |
| /*segment_sizes=*/NULL, operands, results), |
| "call to import '%s'", |
| iree_vm_ImportFunctionDef_full_name(import_def)); |
| } else { |
| VM_VerifyFunctionOrdinal(callee_ordinal); |
| IREE_RETURN_IF_ERROR( |
| iree_vm_bytecode_function_verify_call( |
| verify_state, |
| iree_vm_FunctionSignatureDef_vec_at( |
| verify_state->function_signatures, callee_ordinal), |
| /*segment_sizes=*/NULL, operands, results), |
| "call to internal function %d", callee_ordinal); |
| } |
| }); |
| |
| VERIFY_OP(CORE, CallVariadic, { |
| VM_VerifyFuncAttr(callee_ordinal); |
| VM_VerifyVariadicOperands(segment_sizes); |
| VM_VerifyVariadicOperandsAny(operands); |
| VM_VerifyVariadicResultsAny(results); |
| if (IREE_UNLIKELY(!VM_IsImportOrdinal(callee_ordinal))) { |
| // Variadic calls are currently only supported for import functions. |
| return iree_make_status( |
| IREE_STATUS_FAILED_PRECONDITION, |
| "variadic calls only supported for internal callees"); |
| } |
| VM_UnmaskImportOrdinal(callee_ordinal); |
| VM_VerifyImportOrdinal(callee_ordinal); |
| iree_vm_ImportFunctionDef_table_t import_def = |
| iree_vm_ImportFunctionDef_vec_at(verify_state->imported_functions, |
| callee_ordinal); |
| IREE_RETURN_IF_ERROR( |
| iree_vm_bytecode_function_verify_call( |
| verify_state, iree_vm_ImportFunctionDef_signature(import_def), |
| segment_sizes, operands, results), |
| "variadic call to import '%s'", |
| iree_vm_ImportFunctionDef_full_name(import_def)); |
| }); |
| |
| VERIFY_OP(CORE, Return, { |
| VM_VerifyVariadicOperandsAny(operands); |
| IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_verify_cconv_registers( |
| verify_state, verify_state->cconv_results, /*segment_sizes=*/NULL, |
| operands)); |
| verify_state->in_block = 0; // terminator |
| }); |
| |
| VERIFY_OP(CORE, Fail, { |
| VM_VerifyOperandRegI32(status); |
| iree_string_view_t message; |
| VM_VerifyStrAttr(message, &message); |
| verify_state->in_block = 0; // terminator |
| }); |
| |
| VERIFY_OP(CORE, ImportResolved, { |
| VM_VerifyFuncAttr(import_ordinal); |
| if (IREE_UNLIKELY(!VM_IsImportOrdinal(import_ordinal))) { |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "function ordinal %u is not an import ordinal", |
| import_ordinal); |
| } |
| VM_UnmaskImportOrdinal(import_ordinal); |
| VM_VerifyImportOrdinal(import_ordinal); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Async/fiber ops |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP(CORE, Yield, { |
| VM_VerifyBranchTarget(dest_pc); |
| VM_VerifyBranchOperands(operands); |
| verify_state->in_block = 0; // terminator |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Debugging |
| //===------------------------------------------------------------------===// |
| |
| VERIFY_OP(CORE, Trace, { |
| iree_string_view_t event_name; |
| VM_VerifyStrAttr(event_name, &event_name); |
| VM_VerifyVariadicOperandsAny(operands); |
| }); |
| |
| VERIFY_OP(CORE, Print, { |
| iree_string_view_t event_name; |
| VM_VerifyStrAttr(event_name, &event_name); |
| VM_VerifyVariadicOperandsAny(operands); |
| }); |
| |
| VERIFY_OP(CORE, Break, { |
| VM_VerifyBranchTarget(dest_pc); |
| VM_VerifyBranchOperands(operands); |
| verify_state->in_block = 0; // terminator |
| }); |
| |
| VERIFY_OP(CORE, CondBreak, { |
| VM_VerifyOperandRegI32(condition); |
| VM_VerifyBranchTarget(dest); |
| VM_VerifyBranchOperands(operands); |
| verify_state->in_block = 0; // terminator |
| }); |
| |
| //===------------------------------------------------------------------===// |
| // Extension trampolines |
| //===------------------------------------------------------------------===// |
| |
| #if IREE_VM_EXT_F32_ENABLE |
| BEGIN_VERIFY_PREFIX(PrefixExtF32, iree_vm_FeatureBits_EXT_F32) |
| |
| //===----------------------------------------------------------------===// |
| // ExtF32: Globals |
| //===----------------------------------------------------------------===// |
| |
| VERIFY_OP(EXT_F32, GlobalLoadF32, { |
| VM_VerifyGlobalAttr(byte_offset); |
| VM_VerifyRwdataOffset(byte_offset, 4); |
| VM_VerifyResultRegF32(value); |
| }); |
| |
| VERIFY_OP(EXT_F32, GlobalStoreF32, { |
| VM_VerifyGlobalAttr(byte_offset); |
| VM_VerifyRwdataOffset(byte_offset, 4); |
| VM_VerifyOperandRegF32(value); |
| }); |
| |
| VERIFY_OP(EXT_F32, GlobalLoadIndirectF32, { |
| VM_VerifyOperandRegI32(byte_offset); |
| // NOTE: we have to verify the offset at runtime. |
| VM_VerifyResultRegF32(value); |
| }); |
| |
| VERIFY_OP(EXT_F32, GlobalStoreIndirectF32, { |
| VM_VerifyOperandRegI32(byte_offset); |
| // NOTE: we have to verify the offset at runtime. |
| VM_VerifyOperandRegF32(value); |
| }); |
| |
| //===----------------------------------------------------------------===// |
| // ExtF32: Constants |
| //===----------------------------------------------------------------===// |
| |
| VERIFY_OP(EXT_F32, ConstF32, { |
| VM_VerifyFloatAttr32(value); |
| VM_VerifyResultRegF32(result); |
| }); |
| |
| VERIFY_OP(EXT_F32, ConstF32Zero, { VM_VerifyResultRegF32(result); }); |
| |
| //===----------------------------------------------------------------===// |
| // ExtF32: Lists |
| //===----------------------------------------------------------------===// |
| |
| VERIFY_OP(EXT_F32, ListGetF32, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyResultRegF32(result); |
| }); |
| |
| VERIFY_OP(EXT_F32, ListSetF32, { |
| VM_VerifyOperandRegRef(list); |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyOperandRegF32(value); |
| }); |
| |
| //===----------------------------------------------------------------===// |
| // ExtF32: Conditional assignment |
| //===----------------------------------------------------------------===// |
| |
| VERIFY_OP(EXT_F32, SelectF32, { |
| VM_VerifyOperandRegI32(condition); |
| VM_VerifyOperandRegF32(true_value); |
| VM_VerifyOperandRegF32(false_value); |
| VM_VerifyResultRegF32(result); |
| }); |
| |
| VERIFY_OP(EXT_F32, SwitchF32, { |
| VM_VerifyOperandRegI32(index); |
| VM_VerifyOperandRegF32(default_value); |
| VM_VerifyVariadicOperandsF32(values); |
| VM_VerifyResultRegF32(result); |
| }); |
| |
| //===----------------------------------------------------------------===// |
| // ExtF32: Native floating-point arithmetic |
| //===----------------------------------------------------------------===// |
| |
| VERIFY_OP_EXT_F32_BINARY_F32(AddF32); |
| VERIFY_OP_EXT_F32_BINARY_F32(SubF32); |
| VERIFY_OP_EXT_F32_BINARY_F32(MulF32); |
| VERIFY_OP_EXT_F32_BINARY_F32(DivF32); |
| VERIFY_OP_EXT_F32_BINARY_F32(RemF32); |
| VERIFY_OP_EXT_F32_TERNARY_F32(FMAF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(AbsF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(NegF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(CeilF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(FloorF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(RoundF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(RoundF32Even); |
| VERIFY_OP_EXT_F32_BINARY_F32(MinF32); |
| VERIFY_OP_EXT_F32_BINARY_F32(MaxF32); |
| |
| VERIFY_OP_EXT_F32_UNARY_F32(AtanF32); |
| VERIFY_OP_EXT_F32_BINARY_F32(Atan2F32); |
| VERIFY_OP_EXT_F32_UNARY_F32(CosF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(SinF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(ExpF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(Exp2F32); |
| VERIFY_OP_EXT_F32_UNARY_F32(ExpM1F32); |
| VERIFY_OP_EXT_F32_UNARY_F32(LogF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(Log10F32); |
| VERIFY_OP_EXT_F32_UNARY_F32(Log1pF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(Log2F32); |
| VERIFY_OP_EXT_F32_BINARY_F32(PowF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(RsqrtF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(SqrtF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(TanhF32); |
| VERIFY_OP_EXT_F32_UNARY_F32(ErfF32); |
| |
| //===----------------------------------------------------------------===// |
| // ExtF32: Casting and type conversion/emulation |
| //===----------------------------------------------------------------===// |
| |
| VERIFY_OP(EXT_F32, CastSI32F32, { |
| VM_VerifyOperandRegI32(operand); |
| VM_VerifyResultRegF32(result); |
| }); |
| VERIFY_OP(EXT_F32, CastUI32F32, { |
| VM_VerifyOperandRegI32(operand); |
| VM_VerifyResultRegF32(result); |
| }); |
| VERIFY_OP(EXT_F32, CastF32SI32, { |
| VM_VerifyOperandRegF32(operand); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(EXT_F32, CastF32UI32, { |
| VM_VerifyOperandRegF32(operand); |
| VM_VerifyResultRegI32(result); |
| }); |
| VERIFY_OP(EXT_F32, BitcastI32F32, { |
| VM_VerifyOperandRegI32(operand); |
| VM_VerifyResultRegF32(result); |
| }); |
| VERIFY_OP(EXT_F32, BitcastF32I32, { |
| VM_VerifyOperandRegF32(operand); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| //===----------------------------------------------------------------===// |
| // ExtF32: Comparison ops |
| //===----------------------------------------------------------------===// |
| |
| #define VERIFY_OP_EXT_F32_CMP_F32(op_name) \ |
| VERIFY_OP(EXT_F32, op_name, { \ |
| VM_VerifyOperandRegF32(lhs); \ |
| VM_VerifyOperandRegF32(rhs); \ |
| VM_VerifyResultRegI32(result); \ |
| }); |
| |
| VERIFY_OP_EXT_F32_CMP_F32(CmpEQF32O); |
| VERIFY_OP_EXT_F32_CMP_F32(CmpEQF32U); |
| VERIFY_OP_EXT_F32_CMP_F32(CmpNEF32O); |
| VERIFY_OP_EXT_F32_CMP_F32(CmpNEF32U); |
| VERIFY_OP_EXT_F32_CMP_F32(CmpLTF32O); |
| VERIFY_OP_EXT_F32_CMP_F32(CmpLTF32U); |
| VERIFY_OP_EXT_F32_CMP_F32(CmpLTEF32O); |
| VERIFY_OP_EXT_F32_CMP_F32(CmpLTEF32U); |
| VERIFY_OP(EXT_F32, CmpNaNF32, { |
| VM_VerifyOperandRegF32(operand); |
| VM_VerifyResultRegI32(result); |
| }); |
| |
| //===----------------------------------------------------------------===// |
| // ExtF32: Buffers |
| //===----------------------------------------------------------------===// |
| |
| VERIFY_OP(EXT_F32, BufferFillF32, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegI64HostSize(length); |
| VM_VerifyOperandRegF32(value); |
| }); |
| |
| VERIFY_OP(EXT_F32, BufferLoadF32, { |
| VM_VerifyOperandRegRef(source_buffer); |
| VM_VerifyOperandRegI64HostSize(source_offset); |
| VM_VerifyResultRegF32(result); |
| }); |
| |
| VERIFY_OP(EXT_F32, BufferStoreF32, { |
| VM_VerifyOperandRegRef(target_buffer); |
| VM_VerifyOperandRegI64HostSize(target_offset); |
| VM_VerifyOperandRegF32(value); |
| }); |
| |
| END_VERIFY_PREFIX(); |
| #else |
| UNHANDLED_VERIFY_PREFIX(PrefixExtF32, iree_vm_FeatureBits_EXT_F32); |
| #endif // IREE_VM_EXT_F32_ENABLE |
| |
| VERIFY_OP(CORE, PrefixExtF64, { |
| IREE_VM_VERIFY_REQUIREMENT(iree_vm_FeatureBits_EXT_F64); |
| return iree_make_status(IREE_STATUS_UNIMPLEMENTED, |
| "EXT_64 not yet implemented"); |
| }); |
| |
| default: |
| return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, |
| "unrecognized opcode %u", bytecode_data[pc - 1]); |
| } |
| |
| *out_next_pc = pc; |
| return iree_ok_status(); |
| } |
