Adding bytecode verification. This moves much of the interpreter checks to an ahead-of-time bytecode verifier. This allows us to share the same verification with JITs and disable it entirely for code size reasons using the `-DIREE_VM_BYTECODE_VERIFICATION_ENABLE=0` compiler flag. Verification is pretty exhaustive but may still need some additions. It's significantly better than before, though, so even if not the final form it's a good step. Simpler verification (and dispatch) will come with a pending bytecode shuffling into instruction classes.
diff --git a/build_tools/cmake/iree_copts.cmake b/build_tools/cmake/iree_copts.cmake index 4e16665..1956b04 100644 --- a/build_tools/cmake/iree_copts.cmake +++ b/build_tools/cmake/iree_copts.cmake
@@ -402,6 +402,7 @@ "-DIREE_HAL_MODULE_STRING_UTIL_ENABLE=0" "-DIREE_HAL_COMMAND_BUFFER_VALIDATION_ENABLE=0" "-DIREE_VM_BACKTRACE_ENABLE=0" + "-DIREE_VM_BYTECODE_VERIFICATION_ENABLE=0" "-DIREE_VM_EXT_F32_ENABLE=0" "-DIREE_VM_EXT_F64_ENABLE=0" )
diff --git a/runtime/bindings/tflite/interpreter.c b/runtime/bindings/tflite/interpreter.c index 4183b34..0aaeb92 100644 --- a/runtime/bindings/tflite/interpreter.c +++ b/runtime/bindings/tflite/interpreter.c
@@ -589,7 +589,7 @@ // remain where they currently are for the next invocation. for (iree_host_size_t i = 0; i < interpreter->model->output_count; ++i) { iree_hal_buffer_t* buffer = (iree_hal_buffer_t*)iree_vm_list_get_ref_deref( - interpreter->output_list, i, iree_hal_buffer_get_descriptor()); + interpreter->output_list, i, &iree_hal_buffer_descriptor); TfLiteTensor* tensor = &interpreter->output_tensors[i]; IREE_RETURN_IF_ERROR(_TfLiteTensorBind(tensor, buffer)); }
diff --git a/runtime/src/iree/base/alignment.h b/runtime/src/iree/base/alignment.h index 7be7ebe..a1179f1 100644 --- a/runtime/src/iree/base/alignment.h +++ b/runtime/src/iree/base/alignment.h
@@ -16,6 +16,7 @@ #include <stdint.h> #include <string.h> +#include "iree/base/attributes.h" #include "iree/base/config.h" #include "iree/base/target_platform.h" @@ -107,6 +108,37 @@ #define iree_sizeof_struct(t) iree_host_align(sizeof(t), iree_max_align_t) //===----------------------------------------------------------------------===// +// Alignment intrinsics +//===----------------------------------------------------------------------===// + +#if IREE_HAVE_BUILTIN(__builtin_unreachable) || defined(__GNUC__) +#define IREE_BUILTIN_UNREACHABLE() __builtin_unreachable() +#elif defined(IREE_COMPILER_MSVC) +#define IREE_BUILTIN_UNREACHABLE() __assume(false) +#else +#define IREE_BUILTIN_UNREACHABLE() ((void)0) +#endif // IREE_HAVE_BUILTIN(__builtin_unreachable) || defined(__GNUC__) + +#if !defined(__cplusplus) +#define IREE_DECLTYPE(v) __typeof__(v) +#else +#define IREE_DECLTYPE(v) decltype(v) +#endif // __cplusplus + +#if IREE_HAVE_BUILTIN(__builtin_assume_aligned) || defined(__GNUC__) +// NOTE: gcc only assumes on the result so we have to reset ptr. +#define IREE_BUILTIN_ASSUME_ALIGNED(ptr, size) \ + (ptr = (IREE_DECLTYPE(ptr))(__builtin_assume_aligned((void*)(ptr), (size)))) +#elif 0 // defined(IREE_COMPILER_MSVC) +#define IREE_BUILTIN_ASSUME_ALIGNED(ptr, size) \ + (__assume((((uintptr_t)(ptr)) & ((1 << (size))) - 1)) == 0) +#else +#define IREE_BUILTIN_ASSUME_ALIGNED(ptr, size) \ + ((((uintptr_t)(ptr) % (size)) == 0) ? (ptr) \ + : (IREE_BUILTIN_UNREACHABLE(), (ptr))) +#endif // IREE_HAVE_BUILTIN(__builtin_assume_aligned) || defined(__GNUC__) + +//===----------------------------------------------------------------------===// // Alignment-safe memory accesses //===----------------------------------------------------------------------===//
diff --git a/runtime/src/iree/base/attributes.h b/runtime/src/iree/base/attributes.h index bd396a9..de74591 100644 --- a/runtime/src/iree/base/attributes.h +++ b/runtime/src/iree/base/attributes.h
@@ -17,12 +17,18 @@ // Any call annotated with this will be relatively stable. // Calls without this are considered private to the IREE implementation and // should not be relied upon. -#ifdef __cplusplus +#if defined(__cplusplus) #define IREE_API_EXPORT extern "C" #else #define IREE_API_EXPORT #endif // __cplusplus +#if defined(__cplusplus) +#define IREE_API_EXPORT_VARIABLE extern "C" +#else +#define IREE_API_EXPORT_VARIABLE extern +#endif // __cplusplus + // Denotes a function pointer that is exposed as part of the IREE API. // Example: // iree_status_t(IREE_API_PTR* some_callback)(int value); @@ -33,13 +39,23 @@ //===----------------------------------------------------------------------===// // Queries for [[attribute]] identifiers in modern compilers. -#ifdef __has_attribute +#if defined(__has_attribute) #define IREE_HAVE_ATTRIBUTE(x) __has_attribute(x) #else #define IREE_HAVE_ATTRIBUTE(x) 0 #endif // __has_attribute //===----------------------------------------------------------------------===// +// IREE_HAVE_BUILTIN +//===----------------------------------------------------------------------===// + +#if defined(__has_builtin) +#define IREE_HAVE_BUILTIN(x) __has_builtin(x) +#else +#define IREE_HAVE_BUILTIN(x) 0 +#endif // __has_builtin + +//===----------------------------------------------------------------------===// // IREE_PRINTF_ATTRIBUTE //===----------------------------------------------------------------------===//
diff --git a/runtime/src/iree/base/config.h b/runtime/src/iree/base/config.h index 8490c16..480ac5a 100644 --- a/runtime/src/iree/base/config.h +++ b/runtime/src/iree/base/config.h
@@ -225,10 +225,10 @@ // Enables disassembly of vm bytecode functions and stderr dumping of execution. // Increases code size quite, lowers VM performance, and is generally unsafe; // include only when debugging or running on trusted inputs. -#ifdef NDEBUG -#define IREE_VM_EXECUTION_TRACING_ENABLE 0 -#else +#ifndef NDEBUG #define IREE_VM_EXECUTION_TRACING_ENABLE 1 +#else +#define IREE_VM_EXECUTION_TRACING_ENABLE 0 #endif // NDEBUG #endif // !IREE_VM_EXECUTION_TRACING_ENABLE @@ -256,7 +256,14 @@ // moderate performance improvement (~10-20%) on very heavy VMVX workloads but // adds 20-30KB to the binary size. #define IREE_VM_BYTECODE_DISPATCH_COMPUTED_GOTO_ENABLE 0 -#endif // IREE_VM_BYTECODE_DISPATCH_COMPUTED_GOTO_ENABLE +#endif // !IREE_VM_BYTECODE_DISPATCH_COMPUTED_GOTO_ENABLE + +#if !defined(IREE_VM_BYTECODE_VERIFICATION_ENABLE) +// Enables verification ensuring input bytecode is well-formed. +// This increases binary size but should be left on in all cases where untrusted +// inputs can be provided. Module metadata is always verified. +#define IREE_VM_BYTECODE_VERIFICATION_ENABLE 1 +#endif // !IREE_VM_BYTECODE_VERIFICATION_ENABLE #if !defined(IREE_VM_EXT_F32_ENABLE) // Enables the 32-bit floating-point instruction extension. @@ -273,6 +280,6 @@ #if !defined(IREE_VM_UBSAN_CHECKABLE_ENABLE) // Exposes VMVX kernels to UBSAN checking, else disable UBSAN checking. #define IREE_VM_UBSAN_CHECKABLE_ENABLE 0 -#endif // IREE_VM_UBSAN_CHECKABLE_ENABLE +#endif // !IREE_VM_UBSAN_CHECKABLE_ENABLE #endif // IREE_BASE_CONFIG_H_
diff --git a/runtime/src/iree/base/internal/debugging.h b/runtime/src/iree/base/internal/debugging.h index 0bf232c..dec24ac 100644 --- a/runtime/src/iree/base/internal/debugging.h +++ b/runtime/src/iree/base/internal/debugging.h
@@ -35,7 +35,7 @@ // We implement this directly in the header with ALWAYS_INLINE so that the // stack doesn't get all messed up. IREE_ATTRIBUTE_ALWAYS_INLINE static inline void iree_debug_break(void) { -#if defined(IREE_COMPILER_HAS_BUILTIN_DEBUG_TRAP) +#if IREE_HAVE_BUILTIN(__builtin_debugtrap) __builtin_debugtrap(); #elif defined(IREE_PLATFORM_WINDOWS) __debugbreak(); @@ -50,7 +50,7 @@ #else // NOTE: this is unrecoverable and debugging cannot continue. __builtin_trap(); -#endif // IREE_COMPILER_HAS_BUILTIN_DEBUG_TRAP +#endif // __builtin_debugtrap } //===----------------------------------------------------------------------===//
diff --git a/runtime/src/iree/base/target_platform.h b/runtime/src/iree/base/target_platform.h index 6d4b659..1abd987 100644 --- a/runtime/src/iree/base/target_platform.h +++ b/runtime/src/iree/base/target_platform.h
@@ -202,16 +202,6 @@ #endif // defined(__has_feature) //============================================================================== -// IREE_COMPILER_HAS_BUILTIN_DEBUG_TRAP -//============================================================================== - -#if defined __has_builtin -#if __has_builtin(__builtin_debugtrap) -#define IREE_COMPILER_HAS_BUILTIN_DEBUG_TRAP 1 -#endif -#endif - -//============================================================================== // IREE_PLATFORM_* //==============================================================================
diff --git a/runtime/src/iree/modules/hal/types.c b/runtime/src/iree/modules/hal/types.c index 7109b8b..ba24f9e 100644 --- a/runtime/src/iree/modules/hal/types.c +++ b/runtime/src/iree/modules/hal/types.c
@@ -7,22 +7,28 @@ #include "iree/modules/hal/types.h" //===----------------------------------------------------------------------===// -// Type registration +// Type wrappers //===----------------------------------------------------------------------===// -static iree_vm_ref_type_descriptor_t iree_hal_allocator_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_buffer_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_buffer_view_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_channel_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_command_buffer_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_descriptor_set_layout_descriptor = - {0}; -static iree_vm_ref_type_descriptor_t iree_hal_device_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_event_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_executable_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_fence_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_pipeline_layout_descriptor = {0}; -static iree_vm_ref_type_descriptor_t iree_hal_semaphore_descriptor = {0}; +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_allocator, iree_hal_allocator_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_buffer, iree_hal_buffer_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_buffer_view, iree_hal_buffer_view_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_channel, iree_hal_channel_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_command_buffer, + iree_hal_command_buffer_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_descriptor_set_layout, + iree_hal_descriptor_set_layout_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_device, iree_hal_device_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_event, iree_hal_event_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_executable, iree_hal_executable_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_pipeline_layout, + iree_hal_pipeline_layout_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_fence, iree_hal_fence_t); +IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_semaphore, iree_hal_semaphore_t); + +//===----------------------------------------------------------------------===// +// Type registration +//===----------------------------------------------------------------------===// #define IREE_VM_REGISTER_HAL_C_TYPE(type, name, destroy_fn, descriptor) \ descriptor.type_name = iree_make_cstring_view(name); \ @@ -112,26 +118,6 @@ return iree_ok_status(); } -//===----------------------------------------------------------------------===// -// Type wrappers -//===----------------------------------------------------------------------===// - -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_allocator, iree_hal_allocator_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_buffer, iree_hal_buffer_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_buffer_view, iree_hal_buffer_view_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_channel, iree_hal_channel_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_command_buffer, - iree_hal_command_buffer_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_descriptor_set_layout, - iree_hal_descriptor_set_layout_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_device, iree_hal_device_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_event, iree_hal_event_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_executable, iree_hal_executable_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_pipeline_layout, - iree_hal_pipeline_layout_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_fence, iree_hal_fence_t); -IREE_VM_DEFINE_TYPE_ADAPTERS(iree_hal_semaphore, iree_hal_semaphore_t); - //===--------------------------------------------------------------------===// // Utilities //===--------------------------------------------------------------------===// @@ -139,7 +125,7 @@ IREE_API_EXPORT iree_hal_buffer_view_t* iree_vm_list_get_buffer_view_assign( const iree_vm_list_t* list, iree_host_size_t i) { return (iree_hal_buffer_view_t*)iree_vm_list_get_ref_deref( - list, i, iree_hal_buffer_view_get_descriptor()); + list, i, &iree_hal_buffer_view_descriptor); } IREE_API_EXPORT iree_hal_buffer_view_t* iree_vm_list_get_buffer_view_retain(
diff --git a/runtime/src/iree/task/topology.h b/runtime/src/iree/task/topology.h index 59edd7b..7a1f568 100644 --- a/runtime/src/iree/task/topology.h +++ b/runtime/src/iree/task/topology.h
@@ -58,7 +58,7 @@ uint8_t group_index; // A name assigned to executor workers used for logging/tracing. - char name[15]; + char name[32 - /*group_index*/ 1]; // Processor index in the cpuinfo set. uint32_t processor_index;
diff --git a/runtime/src/iree/vm/buffer.c b/runtime/src/iree/vm/buffer.c index 1882126..0cb2974 100644 --- a/runtime/src/iree/vm/buffer.c +++ b/runtime/src/iree/vm/buffer.c
@@ -12,8 +12,6 @@ #include "iree/base/tracing.h" #include "iree/vm/instance.h" -static iree_vm_ref_type_descriptor_t iree_vm_buffer_descriptor = {0}; - IREE_VM_DEFINE_TYPE_ADAPTERS(iree_vm_buffer, iree_vm_buffer_t); static iree_status_t iree_vm_buffer_map(const iree_vm_buffer_t* buffer, @@ -141,7 +139,7 @@ // Try to map the source buffer first; no use continuing if we can't read the // data to clone. - iree_const_byte_span_t source_span; + iree_const_byte_span_t source_span = iree_const_byte_span_empty(); IREE_RETURN_AND_END_ZONE_IF_ERROR( z0, iree_vm_buffer_map_ro(source_buffer, source_offset, length, 1, &source_span)); @@ -192,10 +190,10 @@ iree_host_size_t length) { IREE_ASSERT_ARGUMENT(source_buffer); IREE_ASSERT_ARGUMENT(target_buffer); - iree_const_byte_span_t source_span; + iree_const_byte_span_t source_span = iree_const_byte_span_empty(); IREE_RETURN_IF_ERROR(iree_vm_buffer_map_ro(source_buffer, source_offset, length, 1, &source_span)); - iree_byte_span_t target_span; + iree_byte_span_t target_span = iree_byte_span_empty(); IREE_RETURN_IF_ERROR(iree_vm_buffer_map_rw(target_buffer, target_offset, length, 1, &target_span)); memcpy(target_span.data, source_span.data, length); @@ -208,10 +206,10 @@ iree_host_size_t length, bool* out_result) { IREE_ASSERT_ARGUMENT(lhs_buffer); IREE_ASSERT_ARGUMENT(rhs_buffer); - iree_const_byte_span_t lhs_span; + iree_const_byte_span_t lhs_span = iree_const_byte_span_empty(); IREE_RETURN_IF_ERROR( iree_vm_buffer_map_ro(lhs_buffer, lhs_offset, length, 1, &lhs_span)); - iree_const_byte_span_t rhs_span; + iree_const_byte_span_t rhs_span = iree_const_byte_span_empty(); IREE_RETURN_IF_ERROR( iree_vm_buffer_map_ro(rhs_buffer, rhs_offset, length, 1, &rhs_span)); *out_result = memcmp(lhs_span.data, rhs_span.data, length) == 0; @@ -230,7 +228,7 @@ iree_host_size_t element_count, iree_host_size_t element_length, const void* value) { IREE_ASSERT_ARGUMENT(target_buffer); - iree_byte_span_t span; + iree_byte_span_t span = iree_byte_span_empty(); IREE_RETURN_IF_ERROR(iree_vm_buffer_map_rw( target_buffer, target_offset * element_length, element_count * element_length, element_length, &span)); @@ -274,7 +272,7 @@ void* target_ptr, iree_host_size_t element_count, iree_host_size_t element_length) { IREE_ASSERT_ARGUMENT(source_buffer); - iree_const_byte_span_t source_span; + iree_const_byte_span_t source_span = iree_const_byte_span_empty(); IREE_RETURN_IF_ERROR(iree_vm_buffer_map_ro( source_buffer, source_offset * element_length, element_count * element_length, element_length, &source_span)); @@ -288,7 +286,7 @@ iree_host_size_t element_length) { IREE_ASSERT_ARGUMENT(source_ptr); IREE_ASSERT_ARGUMENT(target_buffer); - iree_byte_span_t target_span; + iree_byte_span_t target_span = iree_byte_span_empty(); IREE_RETURN_IF_ERROR(iree_vm_buffer_map_rw( target_buffer, target_offset * element_length, element_count * element_length, element_length, &target_span)); @@ -301,7 +299,6 @@ // Already registered. return iree_ok_status(); } - iree_vm_buffer_descriptor.destroy = iree_vm_buffer_destroy; iree_vm_buffer_descriptor.offsetof_counter = offsetof(iree_vm_buffer_t, ref_object.counter);
diff --git a/runtime/src/iree/vm/bytecode/BUILD b/runtime/src/iree/vm/bytecode/BUILD index 54257a3..69bd228 100644 --- a/runtime/src/iree/vm/bytecode/BUILD +++ b/runtime/src/iree/vm/bytecode/BUILD
@@ -22,48 +22,30 @@ iree_runtime_cc_library( name = "module", srcs = [ + "archive.c", "disassembler.c", "disassembler.h", "dispatch.c", "dispatch_util.h", - "generated/op_table.h", "module.c", "module_impl.h", + "verifier.c", + "verifier.h", ], hdrs = [ + "archive.h", "module.h", ], deps = [ "//runtime/src/iree/base", - "//runtime/src/iree/base:core_headers", "//runtime/src/iree/base:tracing", "//runtime/src/iree/base/internal", - "//runtime/src/iree/base/internal/flatcc:parsing", - "//runtime/src/iree/schemas:bytecode_module_def_c_fbs", "//runtime/src/iree/vm", "//runtime/src/iree/vm:ops", + "//runtime/src/iree/vm/bytecode/utils", ], ) -# TODO(#357): Add a script to update op_table.h. -# iree_gentbl_cc_library( -# name = "op_table_gen", -# tbl_outs = [ -# (["--gen-iree-vm-op-table-defs"], "op_table.h"), -# ], -# tblgen = "//tools:iree-tblgen", -# td_file = "//compiler/src/iree/compiler/Dialect/VM/IR:VMOps.td", -# deps = [ -# "//compiler/src/iree/compiler/Dialect/Util/IR:td_files", -# "//compiler/src/iree/compiler/Dialect/VM/IR:td_files", -# "@llvm-project//mlir:CallInterfacesTdFiles", -# "@llvm-project//mlir:ControlFlowInterfacesTdFiles", -# "@llvm-project//mlir:FunctionInterfacesTdFiles", -# "@llvm-project//mlir:OpBaseTdFiles", -# "@llvm-project//mlir:SideEffectInterfacesTdFiles", -# ], -# ) - iree_cmake_extra_content( content = """ if(IREE_BUILD_COMPILER)
diff --git a/runtime/src/iree/vm/bytecode/CMakeLists.txt b/runtime/src/iree/vm/bytecode/CMakeLists.txt index 7e5a59b..08dc9b7 100644 --- a/runtime/src/iree/vm/bytecode/CMakeLists.txt +++ b/runtime/src/iree/vm/bytecode/CMakeLists.txt
@@ -14,23 +14,24 @@ NAME module HDRS + "archive.h" "module.h" SRCS + "archive.c" "disassembler.c" "disassembler.h" "dispatch.c" "dispatch_util.h" - "generated/op_table.h" "module.c" "module_impl.h" + "verifier.c" + "verifier.h" DEPS iree::base - iree::base::core_headers iree::base::internal - iree::base::internal::flatcc::parsing iree::base::tracing - iree::schemas::bytecode_module_def_c_fbs iree::vm + iree::vm::bytecode::utils iree::vm::ops PUBLIC )
diff --git a/runtime/src/iree/vm/bytecode/archive.c b/runtime/src/iree/vm/bytecode/archive.c new file mode 100644 index 0000000..6ee7cdc --- /dev/null +++ b/runtime/src/iree/vm/bytecode/archive.c
@@ -0,0 +1,140 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "iree/vm/bytecode/archive.h" + +#include "iree/vm/bytecode/utils/isa.h" + +// ZIP local file header (comes immediately before each file in the archive). +// In order to find the starting offset of the FlatBuffer in a polyglot archive +// we need to parse this given the variable-length nature of it (we want to +// be robust to file name and alignment changes). +// +// NOTE: all fields are little-endian. +// NOTE: we don't care about the actual module size here; since we support +// streaming archives trying to recover it would require much more +// involved processing (we'd need to reference the central directory). +// If we wanted to support users repacking ZIPs we'd probably want to +// rewrite everything as we store offsets in the FlatBuffer that are +// difficult to update after the archive has been produced. +#define ZIP_LOCAL_FILE_HEADER_SIGNATURE 0x04034B50u +#if defined(IREE_COMPILER_MSVC) +#pragma pack(push, 1) +#endif // IREE_COMPILER_MSVC +typedef struct { + uint32_t signature; // ZIP_LOCAL_FILE_HEADER_SIGNATURE + uint16_t version; + uint16_t general_purpose_flag; + uint16_t compression_method; + uint16_t last_modified_time; + uint16_t last_modified_date; + uint32_t crc32; // 0 for us + uint32_t compressed_size; // 0 for us + uint32_t uncompressed_size; // 0 for us + uint16_t file_name_length; + uint16_t extra_field_length; + // file name (variable size) + // extra field (variable size) +} IREE_ATTRIBUTE_PACKED zip_local_file_header_t; +#if defined(IREE_COMPILER_MSVC) +#pragma pack(pop) +#endif // IREE_COMPILER_MSVC +static_assert(sizeof(zip_local_file_header_t) == 30, "bad packing"); +#if !defined(IREE_ENDIANNESS_LITTLE) || !IREE_ENDIANNESS_LITTLE +#error "little endian required for zip header parsing" +#endif // IREE_ENDIANNESS_LITTLE + +// Strips any ZIP local file header from |contents| and stores the remaining +// range in |out_stripped|. +static iree_status_t iree_vm_bytecode_module_strip_zip_header( + iree_const_byte_span_t contents, iree_const_byte_span_t* out_stripped) { + // Ensure there's at least some bytes we can check for the header. + // Since we're only looking to strip zip stuff here we can check on that. + if (!contents.data || + contents.data_length < sizeof(zip_local_file_header_t)) { + memmove(out_stripped, &contents, sizeof(contents)); + return iree_ok_status(); + } + + // Check to see if there's a zip local header signature. + // For a compliant zip file this is expected to start at offset 0. + const zip_local_file_header_t* header = + (const zip_local_file_header_t*)contents.data; + if (header->signature != ZIP_LOCAL_FILE_HEADER_SIGNATURE) { + // No signature found, probably not a ZIP. + memmove(out_stripped, &contents, sizeof(contents)); + return iree_ok_status(); + } + + // Compute the starting offset of the file. + // Note that we still don't know (or care) if it's the file we want; actual + // FlatBuffer verification happens later on. + uint32_t offset = + sizeof(*header) + header->file_name_length + header->extra_field_length; + if (offset > contents.data_length) { + // Is a ZIP but doesn't have enough data; error out with something more + // useful than the FlatBuffer verification failing later on given that here + // we know this isn't a FlatBuffer. + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "archive self-reports as a zip but does not have " + "enough data to contain a module"); + } + + *out_stripped = iree_make_const_byte_span(contents.data + offset, + contents.data_length - offset); + return iree_ok_status(); +} + +IREE_API_EXPORT iree_status_t iree_vm_bytecode_archive_parse_header( + iree_const_byte_span_t archive_contents, + iree_const_byte_span_t* out_flatbuffer_contents, + iree_host_size_t* out_rodata_offset) { + // Slice off any polyglot zip header we have prior to the base of the module. + iree_const_byte_span_t module_contents = iree_const_byte_span_empty(); + IREE_RETURN_IF_ERROR(iree_vm_bytecode_module_strip_zip_header( + archive_contents, &module_contents)); + + // Verify there's enough data to safely check the FlatBuffer header. + if (!module_contents.data || module_contents.data_length < 16) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "FlatBuffer data is not present or less than 16 bytes (%zu total)", + module_contents.data_length); + } + + // Read the size prefix from the head of the module contents; this should be + // a 4 byte value indicating the total size of the FlatBuffer data. + size_t length_prefix = 0; + flatbuffers_read_size_prefix((void*)module_contents.data, &length_prefix); + + // Verify the length prefix is within bounds (always <= the remaining module + // bytes). + size_t length_remaining = + module_contents.data_length - sizeof(flatbuffers_uoffset_t); + if (length_prefix > length_remaining) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "FlatBuffer length prefix out of bounds (prefix is " + "%zu but only %zu available)", + length_prefix, length_remaining); + } + + // Form the range of bytes containing just the FlatBuffer data. + iree_const_byte_span_t flatbuffer_contents = iree_make_const_byte_span( + module_contents.data + sizeof(flatbuffers_uoffset_t), length_prefix); + + if (out_flatbuffer_contents) { + *out_flatbuffer_contents = flatbuffer_contents; + } + if (out_rodata_offset) { + // rodata begins immediately following the FlatBuffer in memory. + iree_host_size_t rodata_offset = iree_host_align( + (iree_host_size_t)(flatbuffer_contents.data - archive_contents.data) + + length_prefix, + IREE_VM_ARCHIVE_SEGMENT_ALIGNMENT); + *out_rodata_offset = rodata_offset; + } + return iree_ok_status(); +}
diff --git a/runtime/src/iree/vm/bytecode/archive.h b/runtime/src/iree/vm/bytecode/archive.h new file mode 100644 index 0000000..ec4b395 --- /dev/null +++ b/runtime/src/iree/vm/bytecode/archive.h
@@ -0,0 +1,35 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_VM_BYTECODE_ARCHIVE_H_ +#define IREE_VM_BYTECODE_ARCHIVE_H_ + +#include "iree/base/api.h" +#include "iree/vm/api.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +// Alignment applied to each segment of the archive. +// All embedded file contents (FlatBuffers, rodata, etc) are aligned to this +// boundary. +#define IREE_VM_ARCHIVE_SEGMENT_ALIGNMENT 64 + +// Parses the module archive header in |archive_contents|. +// The subrange containing the FlatBuffer data is returned as well as the +// offset where external rodata begins. Note that archives may have +// non-contiguous layouts! +IREE_API_EXPORT iree_status_t iree_vm_bytecode_archive_parse_header( + iree_const_byte_span_t archive_contents, + iree_const_byte_span_t* out_flatbuffer_contents, + iree_host_size_t* out_rodata_offset); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // IREE_VM_BYTECODE_ARCHIVE_H_
diff --git a/runtime/src/iree/vm/bytecode/disassembler.c b/runtime/src/iree/vm/bytecode/disassembler.c index 1b38ec7..e0fbab0 100644 --- a/runtime/src/iree/vm/bytecode/disassembler.c +++ b/runtime/src/iree/vm/bytecode/disassembler.c
@@ -8,7 +8,6 @@ #include <inttypes.h> -#include "iree/base/config.h" #include "iree/vm/ops.h" #define BEGIN_DISASM_PREFIX(op_name, ext) \ @@ -64,39 +63,39 @@ #define VM_ParseBranchOperands(operands_name) \ VM_DecBranchOperandsImpl(bytecode_data, &pc) #define VM_ParseOperandRegI32(name) \ - OP_I16(0) & regs->i32_mask; \ - pc += kRegSize; -#define VM_ParseOperandRegI64(name) \ - OP_I16(0) & (regs->i32_mask & ~1); \ - pc += kRegSize; + OP_I16(0); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_ParseOperandRegI64(name) \ + OP_I16(0); \ + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_ParseOperandRegF32(name) \ - OP_I16(0) & regs->i32_mask; \ - pc += kRegSize; -#define VM_ParseOperandRegF64(name) \ - OP_I16(0) & (regs->i32_mask & ~1); \ - pc += kRegSize; + OP_I16(0); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_ParseOperandRegF64(name) \ + OP_I16(0); \ + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_ParseOperandRegRef(name, out_is_move) \ - OP_I16(0) & regs->ref_mask; \ + OP_I16(0) & IREE_REF_REGISTER_MASK; \ *(out_is_move) = 0; /*= OP_I16(0) & IREE_REF_REGISTER_MOVE_BIT;*/ \ - pc += kRegSize; + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_ParseVariadicOperands(name) \ VM_DecVariadicOperandsImpl(bytecode_data, &pc) #define VM_ParseResultRegI32(name) \ - OP_I16(0) & regs->i32_mask; \ - pc += kRegSize; -#define VM_ParseResultRegI64(name) \ - OP_I16(0) & (regs->i32_mask & ~1); \ - pc += kRegSize; + OP_I16(0); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_ParseResultRegI64(name) \ + OP_I16(0); \ + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_ParseResultRegF32(name) \ - OP_I16(0) & regs->i32_mask; \ - pc += kRegSize; -#define VM_ParseResultRegF64(name) \ - OP_I16(0) & (regs->i32_mask & ~1); \ - pc += kRegSize; + OP_I16(0); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_ParseResultRegF64(name) \ + OP_I16(0); \ + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_ParseResultRegRef(name, out_is_move) \ - OP_I16(0) & regs->ref_mask; \ + OP_I16(0) & IREE_REF_REGISTER_MASK; \ *(out_is_move) = 0; /*= OP_I16(0) & IREE_REF_REGISTER_MOVE_BIT;*/ \ - pc += kRegSize; + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_ParseVariadicResults(name) VM_ParseVariadicOperands(name) #define EMIT_REG_NAME(reg) \ @@ -1320,6 +1319,7 @@ DISASM_OP_CORE_BINARY_I32(RemI32S, "vm.rem.i32.s"); DISASM_OP_CORE_BINARY_I32(RemI32U, "vm.rem.i32.u"); DISASM_OP_CORE_TERNARY_I32(FMAI32, "vm.fma.i32"); + DISASM_OP_CORE_UNARY_I32(AbsI32, "vm.abs.i32"); DISASM_OP_CORE_UNARY_I32(NotI32, "vm.not.i32"); DISASM_OP_CORE_BINARY_I32(AndI32, "vm.and.i32"); DISASM_OP_CORE_BINARY_I32(OrI32, "vm.or.i32"); @@ -1334,6 +1334,7 @@ DISASM_OP_CORE_BINARY_I64(RemI64S, "vm.rem.i64.s"); DISASM_OP_CORE_BINARY_I64(RemI64U, "vm.rem.i64.u"); DISASM_OP_CORE_TERNARY_I64(FMAI64, "vm.fma.i64"); + DISASM_OP_CORE_UNARY_I64(AbsI64, "vm.abs.i64"); DISASM_OP_CORE_UNARY_I64(NotI64, "vm.not.i64"); DISASM_OP_CORE_BINARY_I64(AndI64, "vm.and.i64"); DISASM_OP_CORE_BINARY_I64(OrI64, "vm.or.i64");
diff --git a/runtime/src/iree/vm/bytecode/disassembler.h b/runtime/src/iree/vm/bytecode/disassembler.h index 0f3dd49..36c00ce 100644 --- a/runtime/src/iree/vm/bytecode/disassembler.h +++ b/runtime/src/iree/vm/bytecode/disassembler.h
@@ -9,7 +9,7 @@ #include <stdio.h> -#include "iree/base/string_builder.h" +#include "iree/base/api.h" #include "iree/vm/api.h" #include "iree/vm/bytecode/dispatch_util.h" #include "iree/vm/bytecode/module_impl.h"
diff --git a/runtime/src/iree/vm/bytecode/dispatch.c b/runtime/src/iree/vm/bytecode/dispatch.c index 592adb3..220f9c5 100644 --- a/runtime/src/iree/vm/bytecode/dispatch.c +++ b/runtime/src/iree/vm/bytecode/dispatch.c
@@ -28,7 +28,7 @@ // no swapping hazards (such as 0->1,1->0). The register allocator in the // compiler should ensure this is the case when it can occur. static void iree_vm_bytecode_dispatch_remap_branch_registers( - const iree_vm_registers_t regs, + int32_t* IREE_RESTRICT regs_i32, iree_vm_ref_t* IREE_RESTRICT regs_ref, const iree_vm_register_remap_list_t* IREE_RESTRICT remap_list) { for (int i = 0; i < remap_list->size; ++i) { // TODO(benvanik): change encoding to avoid this branching. @@ -37,10 +37,10 @@ uint16_t dst_reg = remap_list->pairs[i].dst_reg; if (src_reg & IREE_REF_REGISTER_TYPE_BIT) { iree_vm_ref_retain_or_move(src_reg & IREE_REF_REGISTER_MOVE_BIT, - ®s.ref[src_reg & regs.ref_mask], - ®s.ref[dst_reg & regs.ref_mask]); + ®s_ref[src_reg & IREE_REF_REGISTER_MASK], + ®s_ref[dst_reg & IREE_REF_REGISTER_MASK]); } else { - regs.i32[dst_reg & regs.i32_mask] = regs.i32[src_reg & regs.i32_mask]; + regs_i32[dst_reg] = regs_i32[src_reg]; } } } @@ -49,14 +49,14 @@ // This can be used to eagerly release resources we don't need and reduces // memory consumption if used effectively prior to yields/waits. static void iree_vm_bytecode_dispatch_discard_registers( - const iree_vm_registers_t regs, + iree_vm_ref_t* IREE_RESTRICT regs_ref, const iree_vm_register_list_t* IREE_RESTRICT reg_list) { for (int i = 0; i < reg_list->size; ++i) { // TODO(benvanik): change encoding to avoid this branching. uint16_t reg = reg_list->registers[i]; if ((reg & (IREE_REF_REGISTER_TYPE_BIT | IREE_REF_REGISTER_MOVE_BIT)) == (IREE_REF_REGISTER_TYPE_BIT | IREE_REF_REGISTER_MOVE_BIT)) { - iree_vm_ref_release(®s.ref[reg & regs.ref_mask]); + iree_vm_ref_release(®s_ref[reg & IREE_REF_REGISTER_MASK]); } } } @@ -65,45 +65,36 @@ // Stack management //===----------------------------------------------------------------------===// -static iree_vm_registers_t iree_vm_bytecode_get_register_storage( +static inline iree_vm_registers_t iree_vm_bytecode_get_register_storage( iree_vm_stack_frame_t* frame) { const iree_vm_bytecode_frame_storage_t* stack_storage = (iree_vm_bytecode_frame_storage_t*)iree_vm_stack_frame_storage(frame); - - // Masks indicate the valid bits of any register value within the range we - // have allocated in the storage. So for 4 registers we'd expect a 0b11 mask. - iree_vm_registers_t registers; - memset(®isters, 0, sizeof(registers)); - registers.i32_mask = (uint16_t)(stack_storage->i32_register_count - ? stack_storage->i32_register_count - 1 - : 0); - registers.ref_mask = (uint16_t)(stack_storage->ref_register_count - ? stack_storage->ref_register_count - 1 - : 0); - - // Register storage immediately follows the stack storage header. - registers.i32 = - (int32_t*)((uintptr_t)stack_storage + stack_storage->i32_register_offset); - registers.ref = (iree_vm_ref_t*)((uintptr_t)stack_storage + - stack_storage->ref_register_offset); - - return registers; + return (iree_vm_registers_t){ + .i32 = (int32_t*)((uintptr_t)stack_storage + + stack_storage->i32_register_offset), + .ref = (iree_vm_ref_t*)((uintptr_t)stack_storage + + stack_storage->ref_register_offset), + }; } // Releases any remaining refs held in the frame storage. static void iree_vm_bytecode_stack_frame_cleanup(iree_vm_stack_frame_t* frame) { - iree_vm_registers_t regs = iree_vm_bytecode_get_register_storage(frame); // TODO(benvanik): allow the VM to elide this when it's known that there are // no more live registers. - for (uint16_t i = 0; i <= regs.ref_mask; ++i) { - iree_vm_ref_t* ref = ®s.ref[i]; + const iree_vm_bytecode_frame_storage_t* stack_storage = + (iree_vm_bytecode_frame_storage_t*)iree_vm_stack_frame_storage(frame); + iree_vm_ref_t* refs = (iree_vm_ref_t*)((uintptr_t)stack_storage + + stack_storage->ref_register_offset); + for (uint16_t i = 0; i < stack_storage->ref_register_count; ++i) { + iree_vm_ref_t* ref = &refs[i]; if (ref->ptr) iree_vm_ref_release(ref); } } static iree_status_t iree_vm_bytecode_function_enter( iree_vm_stack_t* stack, const iree_vm_function_t function, - iree_string_view_t cconv_results, iree_vm_stack_frame_t** out_callee_frame, + iree_string_view_t cconv_results, + iree_vm_stack_frame_t* IREE_RESTRICT* out_callee_frame, iree_vm_registers_t* out_callee_registers) { iree_vm_bytecode_module_t* module = (iree_vm_bytecode_module_t*)function.module->self; @@ -118,26 +109,11 @@ // bounds check register access. This lets us allocate the entire frame // (header, frame, and register storage) as a single pointer bump below. - // Round up register counts to the nearest power of 2 (if not already). - // This let's us use bit masks on register accesses to do bounds checking - // instead of more complex logic. The cost of these extra registers is only at - // worst 2x the required cost: so not large when thinking about the normal - // size of data used in an IREE app for tensors. - // - // Note that to allow the masking to work as a guard we need to ensure we at - // least allocate 1 register; this way an i32[reg & mask] will always point at - // valid memory even if mask == 0. - uint32_t i32_register_count = iree_math_round_up_to_pow2_u32( - VMMAX(1, target_descriptor->i32_register_count)); - uint32_t ref_register_count = iree_math_round_up_to_pow2_u32( - VMMAX(1, target_descriptor->ref_register_count)); - if (IREE_UNLIKELY(i32_register_count > IREE_I32_REGISTER_MASK) || - IREE_UNLIKELY(ref_register_count > IREE_REF_REGISTER_MASK)) { - // Register count overflow. A valid compiler should never produce files that - // hit this. - return iree_make_status(IREE_STATUS_RESOURCE_EXHAUSTED, - "register count overflow"); - } + // We've verified all register storage prior to execution. + uint32_t i32_register_count = target_descriptor->i32_register_count; + uint32_t ref_register_count = target_descriptor->ref_register_count; + IREE_ASSERT_LE(i32_register_count, IREE_I32_REGISTER_MASK); + IREE_ASSERT_LE(ref_register_count, IREE_REF_REGISTER_MASK); // We need to align the ref register start to the natural machine // alignment in case the compiler is expecting that (it makes it easier to @@ -162,8 +138,8 @@ *out_callee_frame); stack_storage->cconv_results = cconv_results; stack_storage->i32_register_count = i32_register_count; - stack_storage->ref_register_count = ref_register_count; stack_storage->i32_register_offset = header_size; + stack_storage->ref_register_count = ref_register_count; stack_storage->ref_register_offset = header_size + i32_register_size; *out_callee_registers = iree_vm_bytecode_get_register_storage(*out_callee_frame); @@ -181,7 +157,8 @@ static iree_status_t iree_vm_bytecode_external_enter( iree_vm_stack_t* stack, const iree_vm_function_t function, iree_string_view_t cconv_arguments, iree_byte_span_t arguments, - iree_string_view_t cconv_results, iree_vm_stack_frame_t** out_callee_frame, + iree_string_view_t cconv_results, + iree_vm_stack_frame_t* IREE_RESTRICT* out_callee_frame, iree_vm_registers_t* out_callee_registers) { // Enter the bytecode function and allocate registers. IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_enter( @@ -199,23 +176,21 @@ case IREE_VM_CCONV_TYPE_I32: case IREE_VM_CCONV_TYPE_F32: { uint16_t dst_reg = i32_reg++; - memcpy(&callee_registers.i32[dst_reg & callee_registers.i32_mask], p, - sizeof(int32_t)); + memcpy(&callee_registers.i32[dst_reg], p, sizeof(int32_t)); p += sizeof(int32_t); } break; case IREE_VM_CCONV_TYPE_I64: case IREE_VM_CCONV_TYPE_F64: { uint16_t dst_reg = i32_reg; i32_reg += 2; - memcpy(&callee_registers.i32[dst_reg & callee_registers.i32_mask], p, - sizeof(int64_t)); + memcpy(&callee_registers.i32[dst_reg], p, sizeof(int64_t)); p += sizeof(int64_t); } break; case IREE_VM_CCONV_TYPE_REF: { uint16_t dst_reg = ref_reg++; iree_vm_ref_move( (iree_vm_ref_t*)p, - &callee_registers.ref[dst_reg & callee_registers.ref_mask]); + &callee_registers.ref[dst_reg & IREE_REF_REGISTER_MASK]); p += sizeof(iree_vm_ref_t); } break; } @@ -248,22 +223,18 @@ break; case IREE_VM_CCONV_TYPE_I32: case IREE_VM_CCONV_TYPE_F32: { - memcpy(p, &callee_registers->i32[src_reg & callee_registers->i32_mask], - sizeof(int32_t)); + memcpy(p, &callee_registers->i32[src_reg], sizeof(int32_t)); p += sizeof(int32_t); } break; case IREE_VM_CCONV_TYPE_I64: case IREE_VM_CCONV_TYPE_F64: { - memcpy( - p, - &callee_registers->i32[src_reg & (callee_registers->i32_mask & ~1)], - sizeof(int64_t)); + memcpy(p, &callee_registers->i32[src_reg], sizeof(int64_t)); p += sizeof(int64_t); } break; case IREE_VM_CCONV_TYPE_REF: { iree_vm_ref_retain_or_move( src_reg & IREE_REF_REGISTER_MOVE_BIT, - &callee_registers->ref[src_reg & callee_registers->ref_mask], + &callee_registers->ref[src_reg & IREE_REF_REGISTER_MASK], (iree_vm_ref_t*)p); p += sizeof(iree_vm_ref_t); } break; @@ -281,7 +252,7 @@ iree_vm_stack_t* stack, iree_vm_module_t* module, int32_t function_ordinal, const iree_vm_register_list_t* IREE_RESTRICT src_reg_list, const iree_vm_register_list_t* IREE_RESTRICT dst_reg_list, - iree_vm_stack_frame_t** out_callee_frame, + iree_vm_stack_frame_t* IREE_RESTRICT* out_callee_frame, iree_vm_registers_t* out_callee_registers) { // Stash the destination register list for result values on the caller. iree_vm_bytecode_frame_storage_t* caller_storage = @@ -317,15 +288,15 @@ uint16_t src_reg = src_reg_list->registers[i]; if (src_reg & IREE_REF_REGISTER_TYPE_BIT) { uint16_t dst_reg = ref_reg_offset++; - memset(&dst_regs->ref[dst_reg & dst_regs->ref_mask], 0, + memset(&dst_regs->ref[dst_reg & IREE_REF_REGISTER_MASK], 0, sizeof(iree_vm_ref_t)); - iree_vm_ref_retain_or_move(src_reg & IREE_REF_REGISTER_MOVE_BIT, - &src_regs.ref[src_reg & src_regs.ref_mask], - &dst_regs->ref[dst_reg & dst_regs->ref_mask]); + iree_vm_ref_retain_or_move( + src_reg & IREE_REF_REGISTER_MOVE_BIT, + &src_regs.ref[src_reg & IREE_REF_REGISTER_MASK], + &dst_regs->ref[dst_reg & IREE_REF_REGISTER_MASK]); } else { uint16_t dst_reg = i32_reg_offset++; - dst_regs->i32[dst_reg & dst_regs->i32_mask] = - src_regs.i32[src_reg & src_regs.i32_mask]; + dst_regs->i32[dst_reg] = src_regs.i32[src_reg]; } } @@ -339,7 +310,7 @@ iree_vm_stack_t* stack, iree_vm_stack_frame_t* callee_frame, const iree_vm_registers_t callee_registers, const iree_vm_register_list_t* IREE_RESTRICT src_reg_list, - iree_vm_stack_frame_t** out_caller_frame, + iree_vm_stack_frame_t* IREE_RESTRICT* out_caller_frame, iree_vm_registers_t* out_caller_registers) { // Remaps registers from source to destination across frames. // Registers from the |src_regs| will be copied/moved to |dst_regs| with the @@ -357,7 +328,7 @@ caller_frame); const iree_vm_register_list_t* dst_reg_list = caller_storage->return_registers; - VMCHECK(src_reg_list->size <= dst_reg_list->size); + IREE_ASSERT_LE(src_reg_list->size, dst_reg_list->size); if (IREE_UNLIKELY(src_reg_list->size > dst_reg_list->size)) { return iree_make_status(IREE_STATUS_FAILED_PRECONDITION, "src/dst reg count mismatch on internal return"); @@ -372,11 +343,10 @@ if (src_reg & IREE_REF_REGISTER_TYPE_BIT) { iree_vm_ref_retain_or_move( src_reg & IREE_REF_REGISTER_MOVE_BIT, - &callee_registers.ref[src_reg & callee_registers.ref_mask], - &caller_registers.ref[dst_reg & caller_registers.ref_mask]); + &callee_registers.ref[src_reg & IREE_REF_REGISTER_MASK], + &caller_registers.ref[dst_reg & IREE_REF_REGISTER_MASK]); } else { - caller_registers.i32[dst_reg & caller_registers.i32_mask] = - callee_registers.i32[src_reg & callee_registers.i32_mask]; + caller_registers.i32[dst_reg] = callee_registers.i32[src_reg]; } } @@ -401,29 +371,25 @@ break; case IREE_VM_CCONV_TYPE_I32: case IREE_VM_CCONV_TYPE_F32: { - memcpy(p, - &caller_registers.i32[src_reg_list->registers[reg_i++] & - caller_registers.i32_mask], + memcpy(p, &caller_registers.i32[src_reg_list->registers[reg_i++]], sizeof(int32_t)); p += sizeof(int32_t); } break; case IREE_VM_CCONV_TYPE_I64: case IREE_VM_CCONV_TYPE_F64: { - memcpy(p, - &caller_registers.i32[src_reg_list->registers[reg_i++] & - (caller_registers.i32_mask & ~1)], + memcpy(p, &caller_registers.i32[src_reg_list->registers[reg_i++]], sizeof(int64_t)); p += sizeof(int64_t); } break; case IREE_VM_CCONV_TYPE_REF: { uint16_t src_reg = src_reg_list->registers[reg_i++]; iree_vm_ref_assign( - &caller_registers.ref[src_reg & caller_registers.ref_mask], + &caller_registers.ref[src_reg & IREE_REF_REGISTER_MASK], (iree_vm_ref_t*)p); p += sizeof(iree_vm_ref_t); } break; case IREE_VM_CCONV_TYPE_SPAN_START: { - VMCHECK(segment_size_list); + IREE_ASSERT(segment_size_list); int32_t span_count = segment_size_list->registers[seg_i]; memcpy(p, &span_count, sizeof(int32_t)); p += sizeof(int32_t); @@ -448,23 +414,21 @@ case IREE_VM_CCONV_TYPE_I32: case IREE_VM_CCONV_TYPE_F32: { memcpy(p, - &caller_registers.i32[src_reg_list->registers[reg_i++] & - caller_registers.i32_mask], + &caller_registers.i32[src_reg_list->registers[reg_i++]], sizeof(int32_t)); p += sizeof(int32_t); } break; case IREE_VM_CCONV_TYPE_I64: case IREE_VM_CCONV_TYPE_F64: { memcpy(p, - &caller_registers.i32[src_reg_list->registers[reg_i++] & - (caller_registers.i32_mask & ~1)], + &caller_registers.i32[src_reg_list->registers[reg_i++]], sizeof(int64_t)); p += sizeof(int64_t); } break; case IREE_VM_CCONV_TYPE_REF: { uint16_t src_reg = src_reg_list->registers[reg_i++]; iree_vm_ref_assign( - &caller_registers.ref[src_reg & caller_registers.ref_mask], + &caller_registers.ref[src_reg & IREE_REF_REGISTER_MASK], (iree_vm_ref_t*)p); p += sizeof(iree_vm_ref_t); } break; @@ -481,7 +445,7 @@ iree_vm_stack_t* stack, const iree_vm_function_call_t call, iree_string_view_t cconv_results, const iree_vm_register_list_t* IREE_RESTRICT dst_reg_list, - iree_vm_stack_frame_t** out_caller_frame, + iree_vm_stack_frame_t* IREE_RESTRICT* out_caller_frame, iree_vm_registers_t* out_caller_registers) { // Call external function. iree_status_t call_status = @@ -517,21 +481,18 @@ break; case IREE_VM_CCONV_TYPE_I32: case IREE_VM_CCONV_TYPE_F32: - memcpy(&caller_registers.i32[dst_reg & caller_registers.i32_mask], p, - sizeof(int32_t)); + memcpy(&caller_registers.i32[dst_reg], p, sizeof(int32_t)); p += sizeof(int32_t); break; case IREE_VM_CCONV_TYPE_I64: case IREE_VM_CCONV_TYPE_F64: - memcpy( - &caller_registers.i32[dst_reg & (caller_registers.i32_mask & ~1)], - p, sizeof(int64_t)); + memcpy(&caller_registers.i32[dst_reg], p, sizeof(int64_t)); p += sizeof(int64_t); break; case IREE_VM_CCONV_TYPE_REF: iree_vm_ref_move( (iree_vm_ref_t*)p, - &caller_registers.ref[dst_reg & caller_registers.ref_mask]); + &caller_registers.ref[dst_reg & IREE_REF_REGISTER_MASK]); p += sizeof(iree_vm_ref_t); break; } @@ -546,26 +507,27 @@ uint32_t import_ordinal, const iree_vm_bytecode_import_t** out_import) { *out_import = NULL; + // Ordinal has been checked as in-bounds during verification. import_ordinal &= 0x7FFFFFFFu; - if (IREE_UNLIKELY(import_ordinal >= module_state->import_count)) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "import ordinal %u out of range", import_ordinal); - } + IREE_ASSERT(import_ordinal < module_state->import_count); const iree_vm_bytecode_import_t* import = &module_state->import_table[import_ordinal]; if (!import->function.module) { +#if IREE_STATUS_MODE iree_vm_function_t decl_function; IREE_RETURN_IF_ERROR(iree_vm_module_lookup_function_by_ordinal( iree_vm_stack_current_frame(stack)->function.module, IREE_VM_FUNCTION_LINKAGE_IMPORT_OPTIONAL, import_ordinal, &decl_function)); iree_string_view_t import_name = iree_vm_function_name(&decl_function); - (void)import_name; return iree_make_status(IREE_STATUS_NOT_FOUND, "optional import `%.*s` (ordinal %u) not resolved", (int)import_name.size, import_name.data, import_ordinal); +#else + return iree_make_status(IREE_STATUS_NOT_FOUND); +#endif // IREE_STATUS_MODE } *out_import = import; @@ -580,7 +542,7 @@ uint32_t import_ordinal, const iree_vm_registers_t caller_registers, const iree_vm_register_list_t* IREE_RESTRICT src_reg_list, const iree_vm_register_list_t* IREE_RESTRICT dst_reg_list, - iree_vm_stack_frame_t** out_caller_frame, + iree_vm_stack_frame_t* IREE_RESTRICT* out_caller_frame, iree_vm_registers_t* out_caller_registers) { // Prepare |call| by looking up the import information. const iree_vm_bytecode_import_t* import = NULL; @@ -617,7 +579,7 @@ const iree_vm_register_list_t* IREE_RESTRICT segment_size_list, const iree_vm_register_list_t* IREE_RESTRICT src_reg_list, const iree_vm_register_list_t* IREE_RESTRICT dst_reg_list, - iree_vm_stack_frame_t** out_caller_frame, + iree_vm_stack_frame_t* IREE_RESTRICT* out_caller_frame, iree_vm_registers_t* out_caller_registers) { // Prepare |call| by looking up the import information. const iree_vm_bytecode_import_t* import = NULL; @@ -692,9 +654,10 @@ } static iree_status_t iree_vm_bytecode_dispatch( - iree_vm_stack_t* stack, iree_vm_bytecode_module_t* module, - iree_vm_stack_frame_t* current_frame, iree_vm_registers_t regs, - iree_byte_span_t call_results) { + iree_vm_stack_t* IREE_RESTRICT stack, + iree_vm_bytecode_module_t* IREE_RESTRICT module, + iree_vm_stack_frame_t* IREE_RESTRICT current_frame, + iree_vm_registers_t regs, iree_byte_span_t call_results) { // When required emit the dispatch tables here referencing the labels we are // defining below. DEFINE_DISPATCH_TABLES(); @@ -712,8 +675,13 @@ module->bytecode_data.data + module->function_descriptor_table[current_frame->function.ordinal] .bytecode_offset; - iree_vm_source_offset_t pc = current_frame->pc; + int32_t* IREE_RESTRICT regs_i32 = regs.i32; + IREE_BUILTIN_ASSUME_ALIGNED(regs_i32, 16); + iree_vm_ref_t* IREE_RESTRICT regs_ref = regs.ref; + IREE_BUILTIN_ASSUME_ALIGNED(regs_ref, 16); + + iree_vm_source_offset_t pc = current_frame->pc; BEGIN_DISPATCH_CORE() { //===------------------------------------------------------------------===// // Globals @@ -721,13 +689,7 @@ DISPATCH_OP(CORE, GlobalLoadI32, { uint32_t byte_offset = VM_DecGlobalAttr("global"); - if (IREE_UNLIKELY(byte_offset >= - module_state->rwdata_storage.data_length)) { - return iree_make_status( - IREE_STATUS_OUT_OF_RANGE, - "global byte_offset out of range: %d (rwdata=%zu)", byte_offset, - module_state->rwdata_storage.data_length); - } + IREE_ASSERT(byte_offset + 4 <= module_state->rwdata_storage.data_length); int32_t* value = VM_DecResultRegI32("value"); const int32_t global_value = vm_global_load_i32(module_state->rwdata_storage.data, byte_offset); @@ -736,13 +698,7 @@ DISPATCH_OP(CORE, GlobalStoreI32, { uint32_t byte_offset = VM_DecGlobalAttr("global"); - if (IREE_UNLIKELY(byte_offset >= - module_state->rwdata_storage.data_length)) { - return iree_make_status( - IREE_STATUS_OUT_OF_RANGE, - "global byte_offset out of range: %d (rwdata=%zu)", byte_offset, - module_state->rwdata_storage.data_length); - } + IREE_ASSERT(byte_offset + 4 <= module_state->rwdata_storage.data_length); int32_t value = VM_DecOperandRegI32("value"); vm_global_store_i32(module_state->rwdata_storage.data, byte_offset, value); @@ -750,7 +706,7 @@ DISPATCH_OP(CORE, GlobalLoadIndirectI32, { uint32_t byte_offset = VM_DecOperandRegI32("global"); - if (IREE_UNLIKELY(byte_offset >= + if (IREE_UNLIKELY(byte_offset + 4 > module_state->rwdata_storage.data_length)) { return iree_make_status( IREE_STATUS_OUT_OF_RANGE, @@ -765,7 +721,7 @@ DISPATCH_OP(CORE, GlobalStoreIndirectI32, { uint32_t byte_offset = VM_DecOperandRegI32("global"); - if (IREE_UNLIKELY(byte_offset >= + if (IREE_UNLIKELY(byte_offset + 4 > module_state->rwdata_storage.data_length)) { return iree_make_status( IREE_STATUS_OUT_OF_RANGE, @@ -779,13 +735,7 @@ DISPATCH_OP(CORE, GlobalLoadI64, { uint32_t byte_offset = VM_DecGlobalAttr("global"); - if (IREE_UNLIKELY(byte_offset >= - module_state->rwdata_storage.data_length)) { - return iree_make_status( - IREE_STATUS_OUT_OF_RANGE, - "global byte_offset out of range: %d (rwdata=%zu)", byte_offset, - module_state->rwdata_storage.data_length); - } + IREE_ASSERT(byte_offset + 8 <= module_state->rwdata_storage.data_length); int64_t* value = VM_DecResultRegI64("value"); const int64_t global_value = vm_global_load_i64(module_state->rwdata_storage.data, byte_offset); @@ -794,13 +744,7 @@ DISPATCH_OP(CORE, GlobalStoreI64, { uint32_t byte_offset = VM_DecGlobalAttr("global"); - if (IREE_UNLIKELY(byte_offset >= - module_state->rwdata_storage.data_length)) { - return iree_make_status( - IREE_STATUS_OUT_OF_RANGE, - "global byte_offset out of range: %d (rwdata=%zu)", byte_offset, - module_state->rwdata_storage.data_length); - } + IREE_ASSERT(byte_offset + 8 <= module_state->rwdata_storage.data_length); int64_t value = VM_DecOperandRegI64("value"); vm_global_store_i64(module_state->rwdata_storage.data, byte_offset, value); @@ -808,7 +752,7 @@ DISPATCH_OP(CORE, GlobalLoadIndirectI64, { uint32_t byte_offset = VM_DecOperandRegI32("global"); - if (IREE_UNLIKELY(byte_offset >= + if (IREE_UNLIKELY(byte_offset + 8 > module_state->rwdata_storage.data_length)) { return iree_make_status( IREE_STATUS_OUT_OF_RANGE, @@ -823,7 +767,7 @@ DISPATCH_OP(CORE, GlobalStoreIndirectI64, { uint32_t byte_offset = VM_DecOperandRegI32("global"); - if (IREE_UNLIKELY(byte_offset >= + if (IREE_UNLIKELY(byte_offset + 8 > module_state->rwdata_storage.data_length)) { return iree_make_status( IREE_STATUS_OUT_OF_RANGE, @@ -837,12 +781,7 @@ DISPATCH_OP(CORE, GlobalLoadRef, { uint32_t global = VM_DecGlobalAttr("global"); - if (IREE_UNLIKELY(global >= module_state->global_ref_count)) { - return iree_make_status( - IREE_STATUS_OUT_OF_RANGE, - "global ref ordinal out of range: %d (table=%zu)", global, - module_state->global_ref_count); - } + IREE_ASSERT(global < module_state->global_ref_count); const iree_vm_type_def_t* type_def = VM_DecTypeOf("value"); bool result_is_move; iree_vm_ref_t* result = VM_DecResultRegRef("value", &result_is_move); @@ -853,12 +792,7 @@ DISPATCH_OP(CORE, GlobalStoreRef, { uint32_t global = VM_DecGlobalAttr("global"); - if (IREE_UNLIKELY(global >= module_state->global_ref_count)) { - return iree_make_status( - IREE_STATUS_OUT_OF_RANGE, - "global ref ordinal out of range: %d (table=%zu)", global, - module_state->global_ref_count); - } + IREE_ASSERT(global < module_state->global_ref_count); const iree_vm_type_def_t* type_def = VM_DecTypeOf("value"); bool value_is_move; iree_vm_ref_t* value = VM_DecOperandRegRef("value", &value_is_move); @@ -933,12 +867,7 @@ DISPATCH_OP(CORE, ConstRefRodata, { uint32_t rodata_ordinal = VM_DecRodataAttr("rodata"); - if (IREE_UNLIKELY(rodata_ordinal >= module_state->rodata_ref_count)) { - return iree_make_status( - IREE_STATUS_OUT_OF_RANGE, - "rodata ref ordinal out of range: %d (table=%zu)", rodata_ordinal, - module_state->rodata_ref_count); - } + IREE_ASSERT(rodata_ordinal < module_state->rodata_ref_count); bool result_is_move; iree_vm_ref_t* result = VM_DecResultRegRef("value", &result_is_move); IREE_RETURN_IF_ERROR(iree_vm_ref_wrap_retain( @@ -1444,7 +1373,7 @@ VM_DecVariadicOperands("values"); int32_t* result = VM_DecResultRegI32("result"); if (index >= 0 && index < value_reg_list->size) { - *result = regs.i32[value_reg_list->registers[index] & regs.i32_mask]; + *result = regs_i32[value_reg_list->registers[index]]; } else { *result = default_value; } @@ -1457,8 +1386,7 @@ VM_DecVariadicOperands("values"); int64_t* result = VM_DecResultRegI64("result"); if (index >= 0 && index < value_reg_list->size) { - *result = - regs.i32[value_reg_list->registers[index] & (regs.i32_mask & ~1)]; + *result = regs_i32[value_reg_list->registers[index]]; } else { *result = default_value; } @@ -1477,8 +1405,8 @@ if (index >= 0 && index < value_reg_list->size) { bool is_move = value_reg_list->registers[index] & IREE_REF_REGISTER_MOVE_BIT; - iree_vm_ref_t* new_value = - ®s.ref[value_reg_list->registers[index] & regs.ref_mask]; + iree_vm_ref_t* new_value = ®s_ref[value_reg_list->registers[index] & + IREE_REF_REGISTER_MASK]; IREE_RETURN_IF_ERROR(iree_vm_ref_retain_or_move_checked( is_move, new_value, type_def->ref_type, result)); } else { @@ -1645,8 +1573,11 @@ int32_t block_pc = VM_DecBranchTarget("dest"); const iree_vm_register_remap_list_t* remap_list = VM_DecBranchOperands("operands"); - pc = block_pc; - iree_vm_bytecode_dispatch_remap_branch_registers(regs, remap_list); + pc = block_pc + IREE_VM_BLOCK_MARKER_SIZE; // skip block marker + if (IREE_UNLIKELY(remap_list->size > 0)) { + iree_vm_bytecode_dispatch_remap_branch_registers(regs_i32, regs_ref, + remap_list); + } }); DISPATCH_OP(CORE, CondBranch, { @@ -1658,12 +1589,17 @@ const iree_vm_register_remap_list_t* false_remap_list = VM_DecBranchOperands("false_operands"); if (condition) { - pc = true_block_pc; - iree_vm_bytecode_dispatch_remap_branch_registers(regs, true_remap_list); + pc = true_block_pc + IREE_VM_BLOCK_MARKER_SIZE; // skip block marker + if (IREE_UNLIKELY(true_remap_list->size > 0)) { + iree_vm_bytecode_dispatch_remap_branch_registers(regs_i32, regs_ref, + true_remap_list); + } } else { - pc = false_block_pc; - iree_vm_bytecode_dispatch_remap_branch_registers(regs, - false_remap_list); + pc = false_block_pc + IREE_VM_BLOCK_MARKER_SIZE; // skip block marker + if (IREE_UNLIKELY(false_remap_list->size > 0)) { + iree_vm_bytecode_dispatch_remap_branch_registers(regs_i32, regs_ref, + false_remap_list); + } } }); @@ -1692,6 +1628,10 @@ bytecode_data = module->bytecode_data.data + module->function_descriptor_table[function_ordinal].bytecode_offset; + regs_i32 = regs.i32; + IREE_BUILTIN_ASSUME_ALIGNED(regs_i32, 16); + regs_ref = regs.ref; + IREE_BUILTIN_ASSUME_ALIGNED(regs_ref, 16); pc = current_frame->pc; } }); @@ -1747,6 +1687,10 @@ module->bytecode_data.data + module->function_descriptor_table[current_frame->function.ordinal] .bytecode_offset; + regs_i32 = regs.i32; + IREE_BUILTIN_ASSUME_ALIGNED(regs_i32, 16); + regs_ref = regs.ref; + IREE_BUILTIN_ASSUME_ALIGNED(regs_ref, 16); pc = current_frame->pc; }); @@ -1765,10 +1709,7 @@ uint32_t function_ordinal = VM_DecFuncAttr("import"); int32_t* result = VM_DecResultRegI32("result"); uint32_t import_ordinal = function_ordinal & 0x7FFFFFFFu; - if (IREE_UNLIKELY(import_ordinal >= module_state->import_count)) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "import ordinal out of range"); - } + IREE_ASSERT(import_ordinal < module_state->import_count); const iree_vm_bytecode_import_t* import = &module_state->import_table[import_ordinal]; *result = import->function.module != NULL ? 1 : 0; @@ -1784,8 +1725,10 @@ int32_t block_pc = VM_DecBranchTarget("dest"); const iree_vm_register_remap_list_t* remap_list = VM_DecBranchOperands("operands"); - iree_vm_bytecode_dispatch_remap_branch_registers(regs, remap_list); - current_frame->pc = block_pc; + iree_vm_bytecode_dispatch_remap_branch_registers(regs_i32, regs_ref, + remap_list); + current_frame->pc = + block_pc + IREE_VM_BLOCK_MARKER_SIZE; // skip block marker // Return magic status code indicating a yield. // This isn't an error, though callers not supporting coroutines will @@ -1803,7 +1746,7 @@ const iree_vm_register_list_t* src_reg_list = VM_DecVariadicOperands("operands"); // TODO(benvanik): trace (if enabled). - iree_vm_bytecode_dispatch_discard_registers(regs, src_reg_list); + iree_vm_bytecode_dispatch_discard_registers(regs_ref, src_reg_list); }); DISPATCH_OP(CORE, Print, { @@ -1812,7 +1755,7 @@ const iree_vm_register_list_t* src_reg_list = VM_DecVariadicOperands("operands"); // TODO(benvanik): print. - iree_vm_bytecode_dispatch_discard_registers(regs, src_reg_list); + iree_vm_bytecode_dispatch_discard_registers(regs_ref, src_reg_list); }); DISPATCH_OP(CORE, Break, { @@ -1820,7 +1763,8 @@ int32_t block_pc = VM_DecBranchTarget("dest"); const iree_vm_register_remap_list_t* remap_list = VM_DecBranchOperands("operands"); - iree_vm_bytecode_dispatch_remap_branch_registers(regs, remap_list); + iree_vm_bytecode_dispatch_remap_branch_registers(regs_i32, regs_ref, + remap_list); pc = block_pc; }); @@ -1832,8 +1776,9 @@ int32_t block_pc = VM_DecBranchTarget("dest"); const iree_vm_register_remap_list_t* remap_list = VM_DecBranchOperands("operands"); - iree_vm_bytecode_dispatch_remap_branch_registers(regs, remap_list); - pc = block_pc; + iree_vm_bytecode_dispatch_remap_branch_registers(regs_i32, regs_ref, + remap_list); + pc = block_pc + IREE_VM_BLOCK_MARKER_SIZE; // skip block marker }); //===------------------------------------------------------------------===// @@ -1848,13 +1793,8 @@ DISPATCH_OP(EXT_F32, GlobalLoadF32, { uint32_t byte_offset = VM_DecGlobalAttr("global"); - if (IREE_UNLIKELY(byte_offset >= - module_state->rwdata_storage.data_length)) { - return iree_make_status( - IREE_STATUS_OUT_OF_RANGE, - "global byte_offset out of range: %d (rwdata=%zu)", byte_offset, - module_state->rwdata_storage.data_length); - } + IREE_ASSERT(byte_offset + 4 <= + module_state->rwdata_storage.data_length); float* value = VM_DecResultRegF32("value"); const float global_value = vm_global_load_f32(module_state->rwdata_storage.data, byte_offset); @@ -1863,13 +1803,8 @@ DISPATCH_OP(EXT_F32, GlobalStoreF32, { uint32_t byte_offset = VM_DecGlobalAttr("global"); - if (IREE_UNLIKELY(byte_offset >= - module_state->rwdata_storage.data_length)) { - return iree_make_status( - IREE_STATUS_OUT_OF_RANGE, - "global byte_offset out of range: %d (rwdata=%zu)", byte_offset, - module_state->rwdata_storage.data_length); - } + IREE_ASSERT(byte_offset + 4 <= + module_state->rwdata_storage.data_length); float value = VM_DecOperandRegF32("value"); vm_global_store_f32(module_state->rwdata_storage.data, byte_offset, value); @@ -1877,7 +1812,7 @@ DISPATCH_OP(EXT_F32, GlobalLoadIndirectF32, { uint32_t byte_offset = VM_DecOperandRegI32("global"); - if (IREE_UNLIKELY(byte_offset >= + if (IREE_UNLIKELY(byte_offset + 4 > module_state->rwdata_storage.data_length)) { return iree_make_status( IREE_STATUS_OUT_OF_RANGE, @@ -1892,7 +1827,7 @@ DISPATCH_OP(EXT_F32, GlobalStoreIndirectF32, { uint32_t byte_offset = VM_DecOperandRegI32("global"); - if (IREE_UNLIKELY(byte_offset >= + if (IREE_UNLIKELY(byte_offset + 4 > module_state->rwdata_storage.data_length)) { return iree_make_status( IREE_STATUS_OUT_OF_RANGE, @@ -1970,8 +1905,7 @@ VM_DecVariadicOperands("values"); float* result = VM_DecResultRegF32("result"); if (index >= 0 && index < value_reg_list->size) { - *result = *((float*)®s.i32[value_reg_list->registers[index] & - (regs.i32_mask & ~1)]); + *result = *((float*)®s_i32[value_reg_list->registers[index]]); } else { *result = default_value; }
diff --git a/runtime/src/iree/vm/bytecode/dispatch_util.h b/runtime/src/iree/vm/bytecode/dispatch_util.h index 503a372..d460df2 100644 --- a/runtime/src/iree/vm/bytecode/dispatch_util.h +++ b/runtime/src/iree/vm/bytecode/dispatch_util.h
@@ -10,11 +10,9 @@ #include <assert.h> #include <string.h> -#include "iree/base/alignment.h" -#include "iree/base/config.h" -#include "iree/base/target_platform.h" -#include "iree/vm/bytecode/generated/op_table.h" +#include "iree/base/api.h" #include "iree/vm/bytecode/module_impl.h" +#include "iree/vm/bytecode/utils/isa.h" //===----------------------------------------------------------------------===// // Shared data structures @@ -63,14 +61,8 @@ // Pointers to typed register storage. typedef struct iree_vm_registers_t { - // Ordinal mask defining which ordinal bits are valid. All i32 indexing must - // be ANDed with this mask. - uint16_t i32_mask; // 16-byte aligned i32 register array. int32_t* i32; - // Ordinal mask defining which ordinal bits are valid. All ref indexing must - // be ANDed with this mask. - uint16_t ref_mask; // Naturally aligned ref register array. iree_vm_ref_t* ref; } iree_vm_registers_t; @@ -86,30 +78,14 @@ // will be stored by callees upon return. const iree_vm_register_list_t* return_registers; - // Counts of each register type rounded up to the next power of two. - iree_host_size_t i32_register_count; - iree_host_size_t ref_register_count; - - // Relative byte offsets from the head of this struct. - iree_host_size_t i32_register_offset; - iree_host_size_t ref_register_offset; + // Counts of each register type and their relative byte offsets from the head + // of this struct. + uint32_t i32_register_count; + uint32_t i32_register_offset; + uint32_t ref_register_count; + uint32_t ref_register_offset; } iree_vm_bytecode_frame_storage_t; -// Interleaved src-dst register sets for branch register remapping. -// This structure is an overlay for the bytecode that is serialized in a -// matching format. -typedef struct iree_vm_register_remap_list_t { - uint16_t size; - struct pair { - uint16_t src_reg; - uint16_t dst_reg; - } pairs[]; -} iree_vm_register_remap_list_t; -static_assert(iree_alignof(iree_vm_register_remap_list_t) == 2, - "Expecting byte alignment (to avoid padding)"); -static_assert(offsetof(iree_vm_register_remap_list_t, pairs) == 2, - "Expect no padding in the struct"); - // Maps a type ID to a type def with clamping for out of bounds values. static inline const iree_vm_type_def_t* iree_vm_map_type( iree_vm_bytecode_module_t* module, int32_t type_id) { @@ -147,27 +123,6 @@ #define IREE_DISPATCH_MODE_SWITCH 1 #endif // IREE_VM_BYTECODE_DISPATCH_COMPUTED_GOTO_ENABLE -#ifndef NDEBUG -#define VMCHECK(expr) assert(expr) -#else -#define VMCHECK(expr) -#endif // NDEBUG - -//===----------------------------------------------------------------------===// -// Bytecode data reading with little-/big-endian support -//===----------------------------------------------------------------------===// - -static const int kRegSize = sizeof(uint16_t); - -// Bytecode data access macros for reading values of a given type from a byte -// offset within the current function. -#define OP_I8(i) iree_unaligned_load_le((uint8_t*)&bytecode_data[pc + (i)]) -#define OP_I16(i) iree_unaligned_load_le((uint16_t*)&bytecode_data[pc + (i)]) -#define OP_I32(i) iree_unaligned_load_le((uint32_t*)&bytecode_data[pc + (i)]) -#define OP_I64(i) iree_unaligned_load_le((uint64_t*)&bytecode_data[pc + (i)]) -#define OP_F32(i) iree_unaligned_load_le((float*)&bytecode_data[pc + (i)]) -#define OP_F64(i) iree_unaligned_load_le((double*)&bytecode_data[pc + (i)]) - //===----------------------------------------------------------------------===// // Utilities matching the tablegen op encoding scheme //===----------------------------------------------------------------------===// @@ -177,9 +132,6 @@ // Each macro will increment the pc by the number of bytes read and as such must // be called in the same order the values are encoded. -#define VM_AlignPC(pc, alignment) \ - (pc) = ((pc) + ((alignment)-1)) & ~((alignment)-1) - #define VM_DecConstI8(name) \ OP_I8(0); \ ++pc; @@ -195,7 +147,6 @@ #define VM_DecConstF64(name) \ OP_F64(0); \ pc += 8; -#define VM_DecOpcode(opcode) VM_DecConstI8(#opcode) #define VM_DecFuncAttr(name) VM_DecConstI32(name) #define VM_DecGlobalAttr(name) VM_DecConstI32(name) #define VM_DecRodataAttr(name) VM_DecConstI32(name) @@ -216,58 +167,62 @@ VM_DecBranchOperandsImpl(bytecode_data, &pc) static inline const iree_vm_register_remap_list_t* VM_DecBranchOperandsImpl( const uint8_t* IREE_RESTRICT bytecode_data, iree_vm_source_offset_t* pc) { - VM_AlignPC(*pc, kRegSize); + VM_AlignPC(*pc, IREE_REGISTER_ORDINAL_SIZE); const iree_vm_register_remap_list_t* list = (const iree_vm_register_remap_list_t*)&bytecode_data[*pc]; - *pc = *pc + kRegSize + list->size * 2 * kRegSize; + *pc = *pc + IREE_REGISTER_ORDINAL_SIZE + + list->size * 2 * IREE_REGISTER_ORDINAL_SIZE; return list; } -#define VM_DecOperandRegI32(name) \ - regs.i32[OP_I16(0) & regs.i32_mask]; \ - pc += kRegSize; -#define VM_DecOperandRegI64(name) \ - *((int64_t*)®s.i32[OP_I16(0) & (regs.i32_mask & ~1)]); \ - pc += kRegSize; +#define VM_DecOperandRegI32(name) \ + regs_i32[OP_I16(0)]; \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_DecOperandRegI64(name) \ + *((int64_t*)®s_i32[OP_I16(0)]); \ + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_DecOperandRegI64HostSize(name) \ (iree_host_size_t) VM_DecOperandRegI64(name) -#define VM_DecOperandRegF32(name) \ - *((float*)®s.i32[OP_I16(0) & regs.i32_mask]); \ - pc += kRegSize; -#define VM_DecOperandRegF64(name) \ - *((double*)®s.i32[OP_I16(0) & (regs.i32_mask & ~1)]); \ - pc += kRegSize; +#define VM_DecOperandRegF32(name) \ + *((float*)®s_i32[OP_I16(0)]); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_DecOperandRegF64(name) \ + *((double*)®s_i32[OP_I16(0)]); \ + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_DecOperandRegRef(name, out_is_move) \ - ®s.ref[OP_I16(0) & regs.ref_mask]; \ + ®s_ref[OP_I16(0) & IREE_REF_REGISTER_MASK]; \ *(out_is_move) = 0; /*= OP_I16(0) & IREE_REF_REGISTER_MOVE_BIT;*/ \ - pc += kRegSize; + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_DecVariadicOperands(name) \ VM_DecVariadicOperandsImpl(bytecode_data, &pc) static inline const iree_vm_register_list_t* VM_DecVariadicOperandsImpl( const uint8_t* IREE_RESTRICT bytecode_data, iree_vm_source_offset_t* pc) { - VM_AlignPC(*pc, kRegSize); + VM_AlignPC(*pc, IREE_REGISTER_ORDINAL_SIZE); const iree_vm_register_list_t* list = (const iree_vm_register_list_t*)&bytecode_data[*pc]; - *pc = *pc + kRegSize + list->size * kRegSize; + *pc = *pc + IREE_REGISTER_ORDINAL_SIZE + + list->size * IREE_REGISTER_ORDINAL_SIZE; return list; } -#define VM_DecResultRegI32(name) \ - ®s.i32[OP_I16(0) & regs.i32_mask]; \ - pc += kRegSize; -#define VM_DecResultRegI64(name) \ - ((int64_t*)®s.i32[OP_I16(0) & (regs.i32_mask & ~1)]); \ - pc += kRegSize; -#define VM_DecResultRegF32(name) \ - ((float*)®s.i32[OP_I16(0) & regs.i32_mask]); \ - pc += kRegSize; -#define VM_DecResultRegF64(name) \ - ((double*)®s.i32[OP_I16(0) & (regs.i32_mask & ~1)]); \ - pc += kRegSize; +#define VM_DecResultRegI32(name) \ + ®s_i32[OP_I16(0)]; \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_DecResultRegI64(name) \ + ((int64_t*)®s_i32[OP_I16(0)]); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_DecResultRegF32(name) \ + ((float*)®s_i32[OP_I16(0)]); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_DecResultRegF64(name) \ + ((double*)®s_i32[OP_I16(0)]); \ + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_DecResultRegRef(name, out_is_move) \ - ®s.ref[OP_I16(0) & regs.ref_mask]; \ + ®s_ref[OP_I16(0) & IREE_REF_REGISTER_MASK]; \ *(out_is_move) = 0; /*= OP_I16(0) & IREE_REF_REGISTER_MOVE_BIT;*/ \ - pc += kRegSize; + pc += IREE_REGISTER_ORDINAL_SIZE; #define VM_DecVariadicResults(name) VM_DecVariadicOperands(name) +#define IREE_VM_BLOCK_MARKER_SIZE 1 + //===----------------------------------------------------------------------===// // Dispatch table structure //===----------------------------------------------------------------------===// @@ -276,13 +231,6 @@ // doesn't support it, though, and there may be other targets (like wasm) that // can only handle the switch-based approach. -// Bytecode data -offset used when looking for the start of the currently -// dispatched instruction: `instruction_start = pc - OFFSET` -#define VM_PC_OFFSET_CORE 1 -#define VM_PC_OFFSET_EXT_I32 2 -#define VM_PC_OFFSET_EXT_F32 2 -#define VM_PC_OFFSET_EXT_F64 2 - #if defined(IREE_DISPATCH_MODE_COMPUTED_GOTO) // Dispatch table mapping 1:1 with bytecode ops. @@ -334,20 +282,20 @@ #define DISPATCH_UNHANDLED_CORE() \ _dispatch_unhandled : { \ - VMCHECK(0); \ + IREE_ASSERT(0); \ return iree_make_status(IREE_STATUS_UNIMPLEMENTED, "unhandled opcode"); \ } #define UNHANDLED_DISPATCH_PREFIX(op_name, ext) \ _dispatch_CORE_##op_name : { \ - VMCHECK(0); \ + IREE_ASSERT(0); \ return iree_make_status(IREE_STATUS_UNIMPLEMENTED, \ "unhandled dispatch extension " #ext); \ } -#define DISPATCH_OP(ext, op_name, body) \ - _dispatch_##ext##_##op_name:; \ - IREE_DISPATCH_TRACE_INSTRUCTION(VM_PC_OFFSET_##ext, #op_name); \ - body; \ +#define DISPATCH_OP(ext, op_name, body) \ + _dispatch_##ext##_##op_name:; \ + IREE_DISPATCH_TRACE_INSTRUCTION(IREE_VM_PC_OFFSET_##ext, #op_name); \ + body; \ goto* kDispatchTable_CORE[bytecode_data[pc++]]; #define BEGIN_DISPATCH_PREFIX(op_name, ext) \ @@ -367,23 +315,25 @@ #define DEFINE_DISPATCH_TABLES() -#define DISPATCH_UNHANDLED_CORE() \ - default: { \ - VMCHECK(0); \ - return iree_make_status(IREE_STATUS_UNIMPLEMENTED, \ - "unhandled core opcode"); \ +#define DISPATCH_UNHANDLED_CORE() \ + default: { \ + IREE_ASSERT(0); \ + IREE_BUILTIN_UNREACHABLE(); /* ok because verified */ \ + return iree_make_status(IREE_STATUS_UNIMPLEMENTED, \ + "unhandled core opcode"); \ } #define UNHANDLED_DISPATCH_PREFIX(op_name, ext) \ case IREE_VM_OP_CORE_##op_name: { \ - VMCHECK(0); \ + IREE_ASSERT(0); \ + IREE_BUILTIN_UNREACHABLE(); /* ok because verified */ \ return iree_make_status(IREE_STATUS_UNIMPLEMENTED, \ "unhandled dispatch extension " #ext); \ } -#define DISPATCH_OP(ext, op_name, body) \ - case IREE_VM_OP_##ext##_##op_name: { \ - IREE_DISPATCH_TRACE_INSTRUCTION(VM_PC_OFFSET_##ext, #op_name); \ - body; \ +#define DISPATCH_OP(ext, op_name, body) \ + case IREE_VM_OP_##ext##_##op_name: { \ + IREE_DISPATCH_TRACE_INSTRUCTION(IREE_VM_PC_OFFSET_##ext, #op_name); \ + body; \ } break; #define BEGIN_DISPATCH_PREFIX(op_name, ext) \
diff --git a/runtime/src/iree/vm/bytecode/module.c b/runtime/src/iree/vm/bytecode/module.c index 83739a8..4d88f4c 100644 --- a/runtime/src/iree/vm/bytecode/module.c +++ b/runtime/src/iree/vm/bytecode/module.c
@@ -10,146 +10,10 @@ #include <stdint.h> #include <string.h> -#include "iree/base/api.h" #include "iree/base/tracing.h" -#include "iree/vm/api.h" +#include "iree/vm/bytecode/archive.h" #include "iree/vm/bytecode/module_impl.h" - -// Alignment applied to each segment of the archive. -// All embedded file contents (FlatBuffers, rodata, etc) are aligned to this -// boundary. -#define IREE_VM_ARCHIVE_SEGMENT_ALIGNMENT 64 - -// ZIP local file header (comes immediately before each file in the archive). -// In order to find the starting offset of the FlatBuffer in a polyglot archive -// we need to parse this given the variable-length nature of it (we want to -// be robust to file name and alignment changes). -// -// NOTE: all fields are little-endian. -// NOTE: we don't care about the actual module size here; since we support -// streaming archives trying to recover it would require much more -// involved processing (we'd need to reference the central directory). -// If we wanted to support users repacking ZIPs we'd probably want to -// rewrite everything as we store offsets in the FlatBuffer that are -// difficult to update after the archive has been produced. -#define ZIP_LOCAL_FILE_HEADER_SIGNATURE 0x04034B50u -#if defined(IREE_COMPILER_MSVC) -#pragma pack(push, 1) -#endif // IREE_COMPILER_MSVC -typedef struct { - uint32_t signature; // ZIP_LOCAL_FILE_HEADER_SIGNATURE - uint16_t version; - uint16_t general_purpose_flag; - uint16_t compression_method; - uint16_t last_modified_time; - uint16_t last_modified_date; - uint32_t crc32; // 0 for us - uint32_t compressed_size; // 0 for us - uint32_t uncompressed_size; // 0 for us - uint16_t file_name_length; - uint16_t extra_field_length; - // file name (variable size) - // extra field (variable size) -} IREE_ATTRIBUTE_PACKED zip_local_file_header_t; -#if defined(IREE_COMPILER_MSVC) -#pragma pack(pop) -#endif // IREE_COMPILER_MSVC -static_assert(sizeof(zip_local_file_header_t) == 30, "bad packing"); -#if !defined(IREE_ENDIANNESS_LITTLE) || !IREE_ENDIANNESS_LITTLE -#error "little endian required for zip header parsing" -#endif // IREE_ENDIANNESS_LITTLE - -// Strips any ZIP local file header from |contents| and stores the remaining -// range in |out_stripped|. -static iree_status_t iree_vm_bytecode_module_strip_zip_header( - iree_const_byte_span_t contents, iree_const_byte_span_t* out_stripped) { - // Ensure there's at least some bytes we can check for the header. - // Since we're only looking to strip zip stuff here we can check on that. - if (!contents.data || - contents.data_length < sizeof(zip_local_file_header_t)) { - memmove(out_stripped, &contents, sizeof(contents)); - return iree_ok_status(); - } - - // Check to see if there's a zip local header signature. - // For a compliant zip file this is expected to start at offset 0. - const zip_local_file_header_t* header = - (const zip_local_file_header_t*)contents.data; - if (header->signature != ZIP_LOCAL_FILE_HEADER_SIGNATURE) { - // No signature found, probably not a ZIP. - memmove(out_stripped, &contents, sizeof(contents)); - return iree_ok_status(); - } - - // Compute the starting offset of the file. - // Note that we still don't know (or care) if it's the file we want; actual - // FlatBuffer verification happens later on. - uint32_t offset = - sizeof(*header) + header->file_name_length + header->extra_field_length; - if (offset > contents.data_length) { - // Is a ZIP but doesn't have enough data; error out with something more - // useful than the FlatBuffer verification failing later on given that here - // we know this isn't a FlatBuffer. - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "archive self-reports as a zip but does not have " - "enough data to contain a module"); - } - - *out_stripped = iree_make_const_byte_span(contents.data + offset, - contents.data_length - offset); - return iree_ok_status(); -} - -IREE_API_EXPORT iree_status_t iree_vm_bytecode_module_parse_header( - iree_const_byte_span_t archive_contents, - iree_const_byte_span_t* out_flatbuffer_contents, - iree_host_size_t* out_rodata_offset) { - // Slice off any polyglot zip header we have prior to the base of the module. - iree_const_byte_span_t module_contents = iree_const_byte_span_empty(); - IREE_RETURN_IF_ERROR(iree_vm_bytecode_module_strip_zip_header( - archive_contents, &module_contents)); - - // Verify there's enough data to safely check the FlatBuffer header. - if (!module_contents.data || module_contents.data_length < 16) { - return iree_make_status( - IREE_STATUS_INVALID_ARGUMENT, - "FlatBuffer data is not present or less than 16 bytes (%zu total)", - module_contents.data_length); - } - - // Read the size prefix from the head of the module contents; this should be - // a 4 byte value indicating the total size of the FlatBuffer data. - size_t length_prefix = 0; - flatbuffers_read_size_prefix((void*)module_contents.data, &length_prefix); - - // Verify the length prefix is within bounds (always <= the remaining module - // bytes). - size_t length_remaining = - module_contents.data_length - sizeof(flatbuffers_uoffset_t); - if (length_prefix > length_remaining) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "FlatBuffer length prefix out of bounds (prefix is " - "%zu but only %zu available)", - length_prefix, length_remaining); - } - - // Form the range of bytes containing just the FlatBuffer data. - iree_const_byte_span_t flatbuffer_contents = iree_make_const_byte_span( - module_contents.data + sizeof(flatbuffers_uoffset_t), length_prefix); - - if (out_flatbuffer_contents) { - *out_flatbuffer_contents = flatbuffer_contents; - } - if (out_rodata_offset) { - // rodata begins immediately following the FlatBuffer in memory. - iree_host_size_t rodata_offset = iree_host_align( - (iree_host_size_t)(flatbuffer_contents.data - archive_contents.data) + - length_prefix, - IREE_VM_ARCHIVE_SEGMENT_ALIGNMENT); - *out_rodata_offset = rodata_offset; - } - return iree_ok_status(); -} +#include "iree/vm/bytecode/verifier.h" // Perform an strcmp between a FlatBuffers string and an IREE string view. static bool iree_vm_flatbuffer_strcmp(flatbuffers_string_t lhs, @@ -236,252 +100,6 @@ return status; } -// clang-format off -static const iree_bitfield_string_mapping_t iree_vm_bytecode_feature_mappings[] = { - {iree_vm_FeatureBits_EXT_F32, IREE_SVL("EXT_F32")}, - {iree_vm_FeatureBits_EXT_F64, IREE_SVL("EXT_F64")}, -}; -// clang-format on - -// Formats a buffer usage bitfield as a string. -// See iree_bitfield_format for usage. -static iree_string_view_t iree_vm_bytecode_features_format( - iree_vm_FeatureBits_enum_t value, iree_bitfield_string_temp_t* out_temp) { - return iree_bitfield_format_inline( - value, IREE_ARRAYSIZE(iree_vm_bytecode_feature_mappings), - iree_vm_bytecode_feature_mappings, out_temp); -} - -static iree_vm_FeatureBits_enum_t iree_vm_bytecode_available_features(void) { - iree_vm_FeatureBits_enum_t result = 0; -#if IREE_VM_EXT_F32_ENABLE - result |= iree_vm_FeatureBits_EXT_F32; -#endif // IREE_VM_EXT_F32_ENABLE -#if IREE_VM_EXT_F64_ENABLE - result |= iree_vm_FeatureBits_EXT_F64; -#endif // IREE_VM_EXT_F64_ENABLE - return result; -} - -// Verifies the structure of the FlatBuffer so that we can avoid doing so during -// runtime. There are still some conditions we must be aware of (such as omitted -// names on functions with internal linkage), however we shouldn't need to -// bounds check anything within the FlatBuffer after this succeeds. -static iree_status_t iree_vm_bytecode_module_flatbuffer_verify( - iree_const_byte_span_t archive_contents, - iree_const_byte_span_t flatbuffer_contents, - iree_host_size_t archive_rodata_offset) { - // Run flatcc generated verification. This ensures all pointers are in-bounds - // and that we can safely walk the file, but not that the actual contents of - // the FlatBuffer meet our expectations. - int verify_ret = iree_vm_BytecodeModuleDef_verify_as_root( - flatbuffer_contents.data, flatbuffer_contents.data_length); - if (verify_ret != flatcc_verify_ok) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "FlatBuffer verification failed: %s", - flatcc_verify_error_string(verify_ret)); - } - - iree_vm_BytecodeModuleDef_table_t module_def = - iree_vm_BytecodeModuleDef_as_root(flatbuffer_contents.data); - - const iree_vm_FeatureBits_enum_t available_features = - iree_vm_bytecode_available_features(); - const iree_vm_FeatureBits_enum_t required_features = - iree_vm_BytecodeModuleDef_requirements(module_def); - if (!iree_all_bits_set(available_features, required_features)) { -#if IREE_STATUS_MODE - const iree_vm_FeatureBits_enum_t needed_features = - required_features & ~available_features; - iree_bitfield_string_temp_t temp0, temp1, temp2; - iree_string_view_t available_features_str = - iree_vm_bytecode_features_format(available_features, &temp0); - iree_string_view_t required_features_str = - iree_vm_bytecode_features_format(required_features, &temp1); - iree_string_view_t needed_features_str = - iree_vm_bytecode_features_format(needed_features, &temp2); - return iree_make_status( - IREE_STATUS_INVALID_ARGUMENT, - "required module features [%.*s] are not available in this runtime " - "configuration; have [%.*s] while module requires [%.*s]", - (int)needed_features_str.size, needed_features_str.data, - (int)available_features_str.size, available_features_str.data, - (int)required_features_str.size, required_features_str.data); -#else - return iree_status_from_code(IREE_STATUS_INVALID_ARGUMENT); -#endif // IREE_STATUS_MODE - } - - flatbuffers_string_t name = iree_vm_BytecodeModuleDef_name(module_def); - if (!flatbuffers_string_len(name)) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "module missing name field"); - } - - iree_vm_TypeDef_vec_t types = iree_vm_BytecodeModuleDef_types(module_def); - for (size_t i = 0; i < iree_vm_TypeDef_vec_len(types); ++i) { - iree_vm_TypeDef_table_t type_def = iree_vm_TypeDef_vec_at(types, i); - if (!type_def) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "types[%zu] missing body", i); - } - flatbuffers_string_t full_name = iree_vm_TypeDef_full_name(type_def); - if (flatbuffers_string_len(full_name) <= 0) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "types[%zu] missing name", i); - } - } - - iree_vm_RodataSegmentDef_vec_t rodata_segments = - iree_vm_BytecodeModuleDef_rodata_segments(module_def); - for (size_t i = 0; i < iree_vm_RodataSegmentDef_vec_len(rodata_segments); - ++i) { - iree_vm_RodataSegmentDef_table_t segment = - iree_vm_RodataSegmentDef_vec_at(rodata_segments, i); - if (iree_vm_RodataSegmentDef_embedded_data_is_present(segment)) { - continue; // embedded data is verified by FlatBuffers - } - uint64_t segment_offset = - iree_vm_RodataSegmentDef_external_data_offset(segment); - uint64_t segment_length = - iree_vm_RodataSegmentDef_external_data_length(segment); - uint64_t segment_end = - archive_rodata_offset + segment_offset + segment_length; - if (segment_end > archive_contents.data_length) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "rodata[%zu] external reference out of range", i); - } - } - - iree_vm_ModuleDependencyDef_vec_t dependencies = - iree_vm_BytecodeModuleDef_dependencies(module_def); - for (size_t i = 0; i < iree_vm_ModuleDependencyDef_vec_len(dependencies); - ++i) { - iree_vm_ModuleDependencyDef_table_t dependency_def = - iree_vm_ModuleDependencyDef_vec_at(dependencies, i); - flatbuffers_string_t module_name = - iree_vm_ModuleDependencyDef_name(dependency_def); - if (flatbuffers_string_len(module_name) == 0) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "dependencies[%zu] has no module name", i); - } - } - - iree_vm_ImportFunctionDef_vec_t imported_functions = - iree_vm_BytecodeModuleDef_imported_functions(module_def); - iree_vm_ExportFunctionDef_vec_t exported_functions = - iree_vm_BytecodeModuleDef_exported_functions(module_def); - iree_vm_FunctionSignatureDef_vec_t function_signatures = - iree_vm_BytecodeModuleDef_function_signatures(module_def); - iree_vm_FunctionDescriptor_vec_t function_descriptors = - iree_vm_BytecodeModuleDef_function_descriptors(module_def); - - if (iree_vm_FunctionSignatureDef_vec_len(function_signatures) != - iree_vm_FunctionDescriptor_vec_len(function_descriptors)) { - return iree_make_status( - IREE_STATUS_INVALID_ARGUMENT, - "function signature and descriptor table length mismatch (%zu vs %zu)", - iree_vm_FunctionSignatureDef_vec_len(function_signatures), - iree_vm_FunctionDescriptor_vec_len(function_descriptors)); - } - - for (size_t i = 0; i < iree_vm_ImportFunctionDef_vec_len(imported_functions); - ++i) { - iree_vm_ImportFunctionDef_table_t import_def = - iree_vm_ImportFunctionDef_vec_at(imported_functions, i); - if (!import_def) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "imports[%zu] missing body", i); - } - flatbuffers_string_t full_name = - iree_vm_ImportFunctionDef_full_name(import_def); - if (!flatbuffers_string_len(full_name)) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "imports[%zu] missing full_name", i); - } - } - - for (size_t i = 0; i < iree_vm_ExportFunctionDef_vec_len(exported_functions); - ++i) { - iree_vm_ExportFunctionDef_table_t export_def = - iree_vm_ExportFunctionDef_vec_at(exported_functions, i); - if (!export_def) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "exports[%zu] missing body", i); - } - flatbuffers_string_t local_name = - iree_vm_ExportFunctionDef_local_name(export_def); - if (!flatbuffers_string_len(local_name)) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "exports[%zu] missing local_name", i); - } - iree_host_size_t internal_ordinal = - iree_vm_ExportFunctionDef_internal_ordinal(export_def); - if (internal_ordinal >= - iree_vm_FunctionDescriptor_vec_len(function_descriptors)) { - return iree_make_status( - IREE_STATUS_INVALID_ARGUMENT, - "exports[%zu] internal_ordinal out of bounds (0 < %zu < %zu)", i, - internal_ordinal, - iree_vm_FunctionDescriptor_vec_len(function_descriptors)); - } - } - - for (size_t i = 0; - i < iree_vm_FunctionSignatureDef_vec_len(function_signatures); ++i) { - iree_vm_FunctionSignatureDef_table_t function_signature = - iree_vm_FunctionSignatureDef_vec_at(function_signatures, i); - if (!function_signature) { - return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, - "function_signatures[%zu] missing body", i); - } - } - - // Verify that we can properly handle the bytecode embedded in the module. - // We require that major versions match and allow loading of older minor - // versions (we keep changes backwards-compatible). - const uint32_t bytecode_version = - iree_vm_BytecodeModuleDef_bytecode_version(module_def); - const uint32_t bytecode_version_major = bytecode_version >> 16; - const uint32_t bytecode_version_minor = bytecode_version & 0xFFFF; - if ((bytecode_version_major != IREE_VM_BYTECODE_VERSION_MAJOR) || - (bytecode_version_minor > IREE_VM_BYTECODE_VERSION_MINOR)) { - return iree_make_status( - IREE_STATUS_INVALID_ARGUMENT, - "bytecode version mismatch; runtime supports %d.%d, module has %d.%d", - IREE_VM_BYTECODE_VERSION_MAJOR, IREE_VM_BYTECODE_VERSION_MINOR, - bytecode_version_major, bytecode_version_minor); - } - - flatbuffers_uint8_vec_t bytecode_data = - iree_vm_BytecodeModuleDef_bytecode_data(module_def); - for (size_t i = 0; - i < iree_vm_FunctionDescriptor_vec_len(function_descriptors); ++i) { - iree_vm_FunctionDescriptor_struct_t function_descriptor = - iree_vm_FunctionDescriptor_vec_at(function_descriptors, i); - if (function_descriptor->bytecode_offset < 0 || - function_descriptor->bytecode_offset + - function_descriptor->bytecode_length > - flatbuffers_uint8_vec_len(bytecode_data)) { - return iree_make_status( - IREE_STATUS_INVALID_ARGUMENT, - "functions[%zu] descriptor bytecode span out of range (0 < %d < %zu)", - i, function_descriptor->bytecode_offset, - flatbuffers_uint8_vec_len(bytecode_data)); - } - if (function_descriptor->i32_register_count > IREE_I32_REGISTER_COUNT || - function_descriptor->ref_register_count > IREE_REF_REGISTER_COUNT) { - return iree_make_status( - IREE_STATUS_INVALID_ARGUMENT, - "functions[%zu] descriptor register count out of range", i); - } - - // TODO(benvanik): run bytecode verifier on contents. - } - - return iree_ok_status(); -} - static iree_status_t iree_vm_bytecode_map_internal_ordinal( iree_vm_bytecode_module_t* module, iree_vm_function_t function, uint16_t* out_ordinal, @@ -1214,7 +832,7 @@ iree_const_byte_span_t flatbuffer_contents = iree_const_byte_span_empty(); iree_host_size_t archive_rodata_offset = 0; IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_vm_bytecode_module_parse_header( + z0, iree_vm_bytecode_archive_parse_header( archive_contents, &flatbuffer_contents, &archive_rodata_offset)); IREE_TRACE_ZONE_BEGIN_NAMED(z1, "iree_vm_bytecode_module_flatbuffer_verify"); @@ -1294,7 +912,23 @@ module->interface.begin_call = iree_vm_bytecode_module_begin_call; module->interface.resume_call = iree_vm_bytecode_module_resume_call; - *out_module = &module->interface; + // Verify functions in the module now that we've verified the metadata that we + // need to do so. + iree_status_t verify_status = iree_ok_status(); +#if IREE_VM_BYTECODE_VERIFICATION_ENABLE + for (uint16_t i = 0; i < module->function_descriptor_count; ++i) { + IREE_TRACE_ZONE_BEGIN_NAMED(z1, "iree_vm_bytecode_function_verify"); + verify_status = iree_vm_bytecode_function_verify(module, i, allocator); + IREE_TRACE_ZONE_END(z1); + if (!iree_status_is_ok(verify_status)) break; + } +#endif // IREE_VM_BYTECODE_VERIFICATION_ENABLE + if (iree_status_is_ok(verify_status)) { + *out_module = &module->interface; + } else { + iree_allocator_free(allocator, module); + } + IREE_TRACE_ZONE_END(z0); - return iree_ok_status(); + return verify_status; }
diff --git a/runtime/src/iree/vm/bytecode/module.h b/runtime/src/iree/vm/bytecode/module.h index 93ebd82..997762f 100644 --- a/runtime/src/iree/vm/bytecode/module.h +++ b/runtime/src/iree/vm/bytecode/module.h
@@ -7,8 +7,6 @@ #ifndef IREE_VM_BYTECODE_MODULE_H_ #define IREE_VM_BYTECODE_MODULE_H_ -#include <stdint.h> - #include "iree/base/api.h" #include "iree/vm/api.h" @@ -25,15 +23,6 @@ iree_allocator_t archive_allocator, iree_allocator_t allocator, iree_vm_module_t** out_module); -// Parses the module archive header in |archive_contents|. -// The subrange containing the FlatBuffer data is returned as well as the -// offset where external rodata begins. Note that archives may have -// non-contiguous layouts! -IREE_API_EXPORT iree_status_t iree_vm_bytecode_module_parse_header( - iree_const_byte_span_t archive_contents, - iree_const_byte_span_t* out_flatbuffer_contents, - iree_host_size_t* out_rodata_offset); - #ifdef __cplusplus } // extern "C" #endif // __cplusplus
diff --git a/runtime/src/iree/vm/bytecode/module_impl.h b/runtime/src/iree/vm/bytecode/module_impl.h index 2b9728d..70c2f3f 100644 --- a/runtime/src/iree/vm/bytecode/module_impl.h +++ b/runtime/src/iree/vm/bytecode/module_impl.h
@@ -10,48 +10,14 @@ #include <stdint.h> #include <string.h> -// VC++ does not have C11's stdalign.h. -#if !defined(_MSC_VER) -#include <stdalign.h> -#endif // _MSC_VER - #include "iree/base/api.h" #include "iree/vm/api.h" - -// NOTE: include order matters: -#include "iree/base/internal/flatcc/parsing.h" -#include "iree/schemas/bytecode_module_def_reader.h" -#include "iree/schemas/bytecode_module_def_verifier.h" +#include "iree/vm/bytecode/utils/isa.h" #ifdef __cplusplus extern "C" { #endif // __cplusplus -#define VMMAX(a, b) (((a) > (b)) ? (a) : (b)) -#define VMMIN(a, b) (((a) < (b)) ? (a) : (b)) - -// Major bytecode version; mismatches on this will fail in either direction. -// This allows coarse versioning of completely incompatible versions. -// Matches BytecodeEncoder::kVersionMajor in the compiler. -#define IREE_VM_BYTECODE_VERSION_MAJOR 14 -// Minor bytecode version; lower versions are allowed to enable newer runtimes -// to load older serialized files when there are backwards-compatible changes. -// Higher versions are disallowed as they occur when new ops are added that -// otherwise cannot be executed by older runtimes. -// Matches BytecodeEncoder::kVersionMinor in the compiler. -#define IREE_VM_BYTECODE_VERSION_MINOR 0 - -// Maximum register count per bank. -// This determines the bits required to reference registers in the VM bytecode. -#define IREE_I32_REGISTER_COUNT 0x7FFF -#define IREE_REF_REGISTER_COUNT 0x7FFF - -#define IREE_I32_REGISTER_MASK 0x7FFF - -#define IREE_REF_REGISTER_TYPE_BIT 0x8000 -#define IREE_REF_REGISTER_MOVE_BIT 0x4000 -#define IREE_REF_REGISTER_MASK 0x3FFF - // A loaded bytecode module. typedef struct iree_vm_bytecode_module_t { // Interface routing to the bytecode module functions.
diff --git a/runtime/src/iree/vm/bytecode/utils/BUILD b/runtime/src/iree/vm/bytecode/utils/BUILD new file mode 100644 index 0000000..c8ec3be --- /dev/null +++ b/runtime/src/iree/vm/bytecode/utils/BUILD
@@ -0,0 +1,66 @@ +# Copyright 2023 The IREE Authors +# +# Licensed under the Apache License v2.0 with LLVM Exceptions. +# See https://llvm.org/LICENSE.txt for license information. +# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +load("//build_tools/bazel:build_defs.oss.bzl", "iree_runtime_cc_library", "iree_runtime_cc_test") + +package( + default_visibility = ["//visibility:public"], + features = ["layering_check"], + licenses = ["notice"], # Apache 2.0 +) + +iree_runtime_cc_library( + name = "utils", + srcs = [ + "block_list.c", + "features.c", + ], + hdrs = [ + "block_list.h", + "features.h", + "generated/op_table.h", + "isa.h", + ], + deps = [ + "//runtime/src/iree/base", + "//runtime/src/iree/base:tracing", + "//runtime/src/iree/base/internal", + "//runtime/src/iree/base/internal/flatcc:parsing", + "//runtime/src/iree/schemas:bytecode_module_def_c_fbs", + "//runtime/src/iree/vm", + ], +) + +# TODO(#357): Add a script to update op_table.h. +# iree_gentbl_cc_library( +# name = "op_table_gen", +# tbl_outs = [ +# (["--gen-iree-vm-op-table-defs"], "op_table.h"), +# ], +# tblgen = "//tools:iree-tblgen", +# td_file = "//compiler/src/iree/compiler/Dialect/VM/IR:VMOps.td", +# deps = [ +# "//compiler/src/iree/compiler/Dialect/Util/IR:td_files", +# "//compiler/src/iree/compiler/Dialect/VM/IR:td_files", +# "@llvm-project//mlir:CallInterfacesTdFiles", +# "@llvm-project//mlir:ControlFlowInterfacesTdFiles", +# "@llvm-project//mlir:FunctionInterfacesTdFiles", +# "@llvm-project//mlir:OpBaseTdFiles", +# "@llvm-project//mlir:SideEffectInterfacesTdFiles", +# ], +# ) + +iree_runtime_cc_test( + name = "block_list_test", + srcs = ["block_list_test.cc"], + deps = [ + ":utils", + "//runtime/src/iree/base", + "//runtime/src/iree/testing:gtest", + "//runtime/src/iree/testing:gtest_main", + "//runtime/src/iree/vm", + ], +)
diff --git a/runtime/src/iree/vm/bytecode/utils/CMakeLists.txt b/runtime/src/iree/vm/bytecode/utils/CMakeLists.txt new file mode 100644 index 0000000..fa539dc --- /dev/null +++ b/runtime/src/iree/vm/bytecode/utils/CMakeLists.txt
@@ -0,0 +1,47 @@ +################################################################################ +# Autogenerated by build_tools/bazel_to_cmake/bazel_to_cmake.py from # +# runtime/src/iree/vm/bytecode/utils/BUILD # +# # +# Use iree_cmake_extra_content from iree/build_defs.oss.bzl to add arbitrary # +# CMake-only content. # +# # +# To disable autogeneration for this file entirely, delete this header. # +################################################################################ + +iree_add_all_subdirs() + +iree_cc_library( + NAME + utils + HDRS + "block_list.h" + "features.h" + "generated/op_table.h" + "isa.h" + SRCS + "block_list.c" + "features.c" + DEPS + iree::base + iree::base::internal + iree::base::internal::flatcc::parsing + iree::base::tracing + iree::schemas::bytecode_module_def_c_fbs + iree::vm + PUBLIC +) + +iree_cc_test( + NAME + block_list_test + SRCS + "block_list_test.cc" + DEPS + ::utils + iree::base + iree::testing::gtest + iree::testing::gtest_main + iree::vm +) + +### BAZEL_TO_CMAKE_PRESERVES_ALL_CONTENT_BELOW_THIS_LINE ###
diff --git a/runtime/src/iree/vm/bytecode/utils/block_list.c b/runtime/src/iree/vm/bytecode/utils/block_list.c new file mode 100644 index 0000000..d892893 --- /dev/null +++ b/runtime/src/iree/vm/bytecode/utils/block_list.c
@@ -0,0 +1,178 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "iree/vm/bytecode/utils/block_list.h" + +#include "iree/base/tracing.h" + +iree_status_t iree_vm_bytecode_block_list_initialize( + uint32_t capacity, iree_allocator_t allocator, + iree_vm_bytecode_block_list_t* out_block_list) { + IREE_TRACE_ZONE_BEGIN(z0); + IREE_ASSERT_ARGUMENT(out_block_list); + + // In case we fail present an empty list. + out_block_list->capacity = 0; + out_block_list->count = 0; + out_block_list->values = NULL; + + // Configure storage either inline if it fits or as a heap allocation. + if (capacity > IREE_ARRAYSIZE(out_block_list->inline_storage)) { + IREE_RETURN_AND_END_ZONE_IF_ERROR( + z0, iree_allocator_malloc(allocator, + sizeof(out_block_list->values[0]) * capacity, + (void**)&out_block_list->values)); + } else { + out_block_list->values = out_block_list->inline_storage; + } + + // Reset state and clear only the blocks we are using. + out_block_list->capacity = capacity; + out_block_list->count = 0; + memset(out_block_list->values, 0, + sizeof(out_block_list->values[0]) * capacity); + + IREE_TRACE_ZONE_END(z0); + return iree_ok_status(); +} + +void iree_vm_bytecode_block_list_deinitialize( + iree_vm_bytecode_block_list_t* block_list, iree_allocator_t allocator) { + if (!block_list) return; + IREE_TRACE_ZONE_BEGIN(z0); + + if (block_list->values != block_list->inline_storage) { + iree_allocator_free(allocator, block_list->values); + } + block_list->capacity = 0; + block_list->count = 0; + block_list->values = NULL; + + IREE_TRACE_ZONE_END(z0); +} + +iree_status_t iree_vm_bytecode_block_list_insert( + iree_vm_bytecode_block_list_t* block_list, uint32_t pc, + iree_vm_bytecode_block_t** out_block) { + IREE_ASSERT_ARGUMENT(block_list); + *out_block = NULL; + + if (IREE_UNLIKELY(pc >= IREE_VM_PC_BLOCK_MAX)) { + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, + "block pc %08X greater than max %08X", pc, + IREE_VM_PC_BLOCK_MAX); + } + + // Try to find the block or the next block greater than it in the list in case + // we need to insert. + iree_host_size_t ordinal = 0; + iree_status_t status = + iree_vm_bytecode_block_list_find(block_list, pc, &ordinal); + if (iree_status_is_ok(status)) { + // Block found. + *out_block = &block_list->values[ordinal]; + return iree_ok_status(); + } + status = iree_status_ignore(status); + + // Not found, need to insert at ordinal point at the next greatest pc. + if (IREE_UNLIKELY(block_list->count + 1 > block_list->capacity)) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "invalid descriptor block count %u; found at least %u blocks", + block_list->capacity, block_list->count + 1); + } + + // Shift list up and declare new block. + if (ordinal != block_list->count) { + memmove(&block_list->values[ordinal + 1], &block_list->values[ordinal], + (block_list->count - ordinal) * sizeof(block_list->values[0])); + } + iree_vm_bytecode_block_t* block = &block_list->values[ordinal]; + block->defined = 0; + block->reserved = 0; + block->pc = pc; + + ++block_list->count; + *out_block = block; + return iree_ok_status(); +} + +// Finds the ordinal of the block with the given |pc| within the block list. +// Note that these ordinals will change with each insertion and this is +// generally only safe to use after the list has been completed. +// If NOT_FOUND then |out_ordinal| will contain the index into the list of where +// the block would be inserted. +iree_status_t iree_vm_bytecode_block_list_find( + const iree_vm_bytecode_block_list_t* block_list, uint32_t pc, + iree_host_size_t* out_ordinal) { + IREE_ASSERT_ARGUMENT(block_list); + *out_ordinal = 0; + int low = 0; + int high = (int)block_list->count - 1; + while (low <= high) { + const int mid = low + (high - low) / 2; + const uint32_t mid_pc = block_list->values[mid].pc; + if (mid_pc < pc) { + low = mid + 1; + } else if (mid_pc > pc) { + high = mid - 1; + } else { + // Found; early exit. + *out_ordinal = mid; + return iree_ok_status(); + } + } + // Not found; return the next highest slot. Note that this may be off the + // end of the list if the search pc is greater than all current values. + *out_ordinal = low; + return iree_status_from_code(IREE_STATUS_NOT_FOUND); +} + +iree_status_t iree_vm_bytecode_block_list_verify( + const iree_vm_bytecode_block_list_t* block_list, + iree_const_byte_span_t bytecode_data) { + IREE_ASSERT_ARGUMENT(block_list); + + // Ensure we have as many blocks as expected. + if (block_list->count != block_list->capacity) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "invalid descriptor block count %u; found %u blocks", + block_list->capacity, block_list->count); + } + + for (uint32_t i = 0; i < block_list->count; ++i) { + const iree_vm_bytecode_block_t* block = &block_list->values[i]; + + // Ensure all blocks are defined. + if (!block->defined) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "block at pc %08X not defined in bytecode", + block->pc); + } + + // Ensure each block pc is in bounds - we need at least 1 byte for the + // marker. + if (block->pc + 1 >= bytecode_data.data_length) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "block at pc %08X (%u) out of bytecode data range %" PRIhsz, + block->pc, block->pc, bytecode_data.data_length); + } + + // Ensure each block has a block opcode at its target. + uint8_t opc = bytecode_data.data[block->pc]; + if (opc != IREE_VM_OP_CORE_Block) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "block at pc %08X does not start with a block marker opcode", + block->pc); + } + } + + return iree_ok_status(); +}
diff --git a/runtime/src/iree/vm/bytecode/utils/block_list.h b/runtime/src/iree/vm/bytecode/utils/block_list.h new file mode 100644 index 0000000..65e6960 --- /dev/null +++ b/runtime/src/iree/vm/bytecode/utils/block_list.h
@@ -0,0 +1,90 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_VM_BYTECODE_UTILS_BLOCK_LIST_H_ +#define IREE_VM_BYTECODE_UTILS_BLOCK_LIST_H_ + +#include "iree/base/api.h" +#include "iree/vm/api.h" +#include "iree/vm/bytecode/utils/isa.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +// Reserved inline storage capacity for blocks in the block list. +// If more blocks than this are requested we'll heap allocate instead. +// Most programs today end up with a small number of blocks (sometimes 2-3 for +// the entire program). +#define IREE_VM_BYTECODE_INLINE_BLOCK_LIST_CAPACITY (32) + +// A tracked block within the block list. +// Blocks may be either declared (observed as a branch target) or defined +// (observed within the bytecode stream). +typedef struct iree_vm_bytecode_block_t { + // Set only if the block definition has been seen. + uint32_t defined : 1; + uint32_t reserved : 7; + // Program counter of the block within the function. + uint32_t pc : 24; +} iree_vm_bytecode_block_t; + +// A sorted list of blocks. +// Allows for single-pass verification +typedef struct iree_vm_bytecode_block_list_t { + // Available capacity in |values| and the total expected block count. + uint32_t capacity; + // Current count of blocks in the |values| list. + uint32_t count; + // List of blocks sorted by program counter. + // Will either point to inline_storage or be a heap allocation. + iree_vm_bytecode_block_t* values; + // Inlined storage for reasonable block counts to avoid the heap alloc. + iree_vm_bytecode_block_t + inline_storage[IREE_VM_BYTECODE_INLINE_BLOCK_LIST_CAPACITY]; +} iree_vm_bytecode_block_list_t; + +// Initializes a block list with the expected count of |capacity|. +// The same |allocator| must be passed to +// iree_vm_bytecode_block_list_deinitialize; the expectation is that the hosting +// data structure already has a reference to the allocator. +iree_status_t iree_vm_bytecode_block_list_initialize( + uint32_t capacity, iree_allocator_t allocator, + iree_vm_bytecode_block_list_t* out_block_list); + +// Deinitializes a block list using |allocator| if any heap allocations were +// required (must be the same as passed to +// iree_vm_bytecode_block_list_initialize). +void iree_vm_bytecode_block_list_deinitialize( + iree_vm_bytecode_block_list_t* block_list, iree_allocator_t allocator); + +// Looks up a block at |pc| in the |block_list|. If not found the block is +// inserted as declared. Returns the block. Fails if capacity is exceeded. +// The returned |out_block| pointer is only valid until the next insertion. +iree_status_t iree_vm_bytecode_block_list_insert( + iree_vm_bytecode_block_list_t* block_list, uint32_t pc, + iree_vm_bytecode_block_t** out_block); + +// Finds the ordinal of the block with the given |pc| within the block list. +// Note that these ordinals will change with each insertion and this is +// generally only safe to use after the list has been completed. +// If NOT_FOUND then |out_ordinal| will contain the index into the list of where +// the block would be inserted. +iree_status_t iree_vm_bytecode_block_list_find( + const iree_vm_bytecode_block_list_t* block_list, uint32_t pc, + iree_host_size_t* out_ordinal); + +// Verifies that all blocks in the block list were defined and have proper +// tracking in |bytecode_data|. +iree_status_t iree_vm_bytecode_block_list_verify( + const iree_vm_bytecode_block_list_t* block_list, + iree_const_byte_span_t bytecode_data); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // IREE_VM_BYTECODE_UTILS_BLOCK_LIST_H_
diff --git a/runtime/src/iree/vm/bytecode/utils/block_list_test.cc b/runtime/src/iree/vm/bytecode/utils/block_list_test.cc new file mode 100644 index 0000000..bba61c8 --- /dev/null +++ b/runtime/src/iree/vm/bytecode/utils/block_list_test.cc
@@ -0,0 +1,313 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "iree/vm/bytecode/utils/block_list.h" + +#include "iree/base/api.h" +#include "iree/testing/gtest.h" +#include "iree/testing/status_matchers.h" + +namespace { + +using iree::Status; +using iree::StatusCode; +using iree::testing::status::IsOkAndHolds; +using iree::testing::status::StatusIs; +using testing::ElementsAre; +using testing::Eq; + +// Tests usage on empty lists. +TEST(BlockListTest, Empty) { + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(0u, allocator, &block_list)); + + // Try finding with an empty list. Note that we expect the ordinal to be valid + // even though we can't insert anything. + iree_host_size_t ordinal = 0; + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_find(&block_list, 0u, &ordinal)), + StatusIs(StatusCode::kNotFound)); + EXPECT_EQ(ordinal, 0); + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_find(&block_list, 123u, &ordinal)), + StatusIs(StatusCode::kNotFound)); + EXPECT_EQ(ordinal, 0); + + // No blocks inserted to verify. + IREE_EXPECT_OK(iree_vm_bytecode_block_list_verify( + &block_list, iree_const_byte_span_empty())); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +// Valid IR usage for 3 blocks. Note that we insert them out of order: 1 2 0. +// These should be stored inline in the block list struct. +TEST(BlockListTest, Valid) { + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(3u, allocator, &block_list)); + + // Try finding blocks before anything is defined. + iree_host_size_t ordinal = 0; + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_find(&block_list, 0u, &ordinal)), + StatusIs(StatusCode::kNotFound)); + EXPECT_EQ(ordinal, 0); + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_find(&block_list, 123u, &ordinal)), + StatusIs(StatusCode::kNotFound)); + EXPECT_EQ(ordinal, 0); + + iree_vm_bytecode_block_t* block = NULL; + + // Define block 1. + block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 1u, &block)); + EXPECT_EQ(block_list.count, 1); + EXPECT_EQ(block->defined, 0); + EXPECT_EQ(block->pc, 1u); + block->defined = 1; + + // Define block 2. + block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 2u, &block)); + EXPECT_EQ(block_list.count, 2); + EXPECT_EQ(block->defined, 0); + EXPECT_EQ(block->pc, 2u); + block->defined = 1; + + // Define block 0. + block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 0u, &block)); + EXPECT_EQ(block_list.count, 3); + EXPECT_EQ(block->defined, 0); + EXPECT_EQ(block->pc, 0u); + block->defined = 1; + + // Re-insert block 1. Should be a no-op as it is defined. + block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 1u, &block)); + EXPECT_EQ(block_list.count, 3); + EXPECT_EQ(block->defined, 1); + EXPECT_EQ(block->pc, 1u); + + // Find each block and ensure they match. + IREE_EXPECT_OK(iree_vm_bytecode_block_list_find(&block_list, 0u, &ordinal)); + EXPECT_EQ(ordinal, 0); + IREE_EXPECT_OK(iree_vm_bytecode_block_list_find(&block_list, 1u, &ordinal)); + EXPECT_EQ(ordinal, 1); + IREE_EXPECT_OK(iree_vm_bytecode_block_list_find(&block_list, 2u, &ordinal)); + EXPECT_EQ(ordinal, 2); + + // Verify blocks are all defined and have block markers. + std::vector<uint8_t> bytecode_data = { + IREE_VM_OP_CORE_Block, IREE_VM_OP_CORE_Block, IREE_VM_OP_CORE_Block, + IREE_VM_OP_CORE_AbsI32, // need at least one op in a block + }; + IREE_EXPECT_OK(iree_vm_bytecode_block_list_verify( + &block_list, + iree_make_const_byte_span(bytecode_data.data(), bytecode_data.size()))); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +// Tests that a declared block that was never defined errors on verification. +TEST(BlockListTest, Undefined) { + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(1u, allocator, &block_list)); + + // Declare the block. + iree_vm_bytecode_block_t* block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 0u, &block)); + + // Fail verification because it hasn't been defined. + std::vector<uint8_t> bytecode_data = { + IREE_VM_OP_CORE_Block, + }; + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_verify( + &block_list, iree_make_const_byte_span(bytecode_data.data(), + bytecode_data.size()))), + StatusIs(StatusCode::kInvalidArgument)); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +// Tests adding fewer blocks than expected by the capacity. +TEST(BlockListTest, Underflow) { + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(2u, allocator, &block_list)); + + // Declaring; OK (count = 1, capacity = 2). + iree_vm_bytecode_block_t* block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 0u, &block)); + + // Defining; OK (no change in count). + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 0u, &block)); + block->defined = 1; + + // Fail verification because we're missing a block. + std::vector<uint8_t> bytecode_data = { + IREE_VM_OP_CORE_Block, + IREE_VM_OP_CORE_AbsI32, // need at least one op in a block + }; + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_verify( + &block_list, iree_make_const_byte_span(bytecode_data.data(), + bytecode_data.size()))), + StatusIs(StatusCode::kInvalidArgument)); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +// Tests adding more blocks than allowed by the capacity. +TEST(BlockListTest, Overflow) { + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(1u, allocator, &block_list)); + + // Declaring; OK (count = 1, capacity = 1). + iree_vm_bytecode_block_t* block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 0u, &block)); + EXPECT_EQ(block_list.count, 1); + + // Defining; OK (no change in count). + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 0u, &block)); + block->defined = 1; + + // Error: too many blocks. + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_insert(&block_list, 1u, &block)), + StatusIs(StatusCode::kInvalidArgument)); + EXPECT_EQ(block_list.count, 1); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +// Tests adding any blocks to an expected-empty list. +TEST(BlockListTest, OverflowEmpty) { + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(0u, allocator, &block_list)); + + // Error: too many blocks. + iree_vm_bytecode_block_t* block = NULL; + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_insert(&block_list, 0u, &block)), + StatusIs(StatusCode::kInvalidArgument)); + EXPECT_EQ(block_list.count, 0); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +// Tests a block that is missing its marker in the bytecode. +TEST(BlockListTest, MissingMarker) { + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(1u, allocator, &block_list)); + + iree_vm_bytecode_block_t* block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 0u, &block)); + block->defined = 1; + + std::vector<uint8_t> bytecode_data = { + IREE_VM_OP_CORE_AbsI32, // *not* the block marker + }; + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_verify( + &block_list, iree_make_const_byte_span(bytecode_data.data(), + bytecode_data.size()))), + StatusIs(StatusCode::kInvalidArgument)); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +// Tests a block with a pc outside of the bytecode range. +TEST(BlockListTest, OutOfBoundsPC) { + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(1u, allocator, &block_list)); + + iree_vm_bytecode_block_t* block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, 1u, &block)); + block->defined = 1; + + std::vector<uint8_t> bytecode_data = { + IREE_VM_OP_CORE_AbsI32, // *not* the block marker + }; + EXPECT_THAT( + Status(iree_vm_bytecode_block_list_verify( + &block_list, iree_make_const_byte_span(bytecode_data.data(), + bytecode_data.size()))), + StatusIs(StatusCode::kInvalidArgument)); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +// Tests inserting a block with a PC outside of what we can track. This should +// be really rare in practice. +TEST(BlockListTest, OverMaxPC) { + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(1u, allocator, &block_list)); + + iree_vm_bytecode_block_t* block = NULL; + EXPECT_THAT(Status(iree_vm_bytecode_block_list_insert( + &block_list, IREE_VM_PC_BLOCK_MAX + 1, &block)), + StatusIs(StatusCode::kOutOfRange)); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +// Tests adding a lot of blocks such that we trigger a heap storage allocation. +TEST(BlockListTest, HeapStorage) { + uint32_t count = IREE_VM_BYTECODE_INLINE_BLOCK_LIST_CAPACITY * 8; + iree_allocator_t allocator = iree_allocator_system(); + iree_vm_bytecode_block_list_t block_list; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_initialize(count, allocator, &block_list)); + + // Declare all blocks in reverse, for fun. + for (uint32_t i = 0; i < count; ++i) { + iree_vm_bytecode_block_t* block = NULL; + IREE_ASSERT_OK( + iree_vm_bytecode_block_list_insert(&block_list, count - i - 1, &block)); + } + + // Ensure sorted. + for (uint32_t i = 0; i < count; ++i) { + EXPECT_EQ(block_list.values[i].pc, i); + } + + // Define all blocks forward. + for (uint32_t i = 0; i < count; ++i) { + iree_vm_bytecode_block_t* block = NULL; + IREE_ASSERT_OK(iree_vm_bytecode_block_list_insert(&block_list, i, &block)); + block->defined = 1; + } + + // Fake block data (+1 trailing block op for padding) to verify. + std::vector<uint8_t> bytecode_data(count + 1, IREE_VM_OP_CORE_Block); + IREE_EXPECT_OK(iree_vm_bytecode_block_list_verify( + &block_list, + iree_make_const_byte_span(bytecode_data.data(), bytecode_data.size()))); + + iree_vm_bytecode_block_list_deinitialize(&block_list, allocator); +} + +} // namespace
diff --git a/runtime/src/iree/vm/bytecode/utils/features.c b/runtime/src/iree/vm/bytecode/utils/features.c new file mode 100644 index 0000000..6a333f9 --- /dev/null +++ b/runtime/src/iree/vm/bytecode/utils/features.c
@@ -0,0 +1,60 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "iree/vm/bytecode/utils/features.h" + +// clang-format off +static const iree_bitfield_string_mapping_t iree_vm_bytecode_feature_mappings[] = { + {iree_vm_FeatureBits_EXT_F32, IREE_SVL("EXT_F32")}, + {iree_vm_FeatureBits_EXT_F64, IREE_SVL("EXT_F64")}, +}; +// clang-format on + +iree_string_view_t iree_vm_bytecode_features_format( + iree_vm_FeatureBits_enum_t value, iree_bitfield_string_temp_t* out_temp) { + return iree_bitfield_format_inline( + value, IREE_ARRAYSIZE(iree_vm_bytecode_feature_mappings), + iree_vm_bytecode_feature_mappings, out_temp); +} + +iree_vm_FeatureBits_enum_t iree_vm_bytecode_available_features(void) { + iree_vm_FeatureBits_enum_t result = 0; +#if IREE_VM_EXT_F32_ENABLE + result |= iree_vm_FeatureBits_EXT_F32; +#endif // IREE_VM_EXT_F32_ENABLE +#if IREE_VM_EXT_F64_ENABLE + result |= iree_vm_FeatureBits_EXT_F64; +#endif // IREE_VM_EXT_F64_ENABLE + return result; +} + +iree_status_t iree_vm_check_feature_mismatch( + const char* file, int line, iree_vm_FeatureBits_enum_t required_features, + iree_vm_FeatureBits_enum_t available_features) { + if (IREE_LIKELY(iree_all_bits_set(available_features, required_features))) { + return iree_ok_status(); + } +#if IREE_STATUS_MODE + const iree_vm_FeatureBits_enum_t needed_features = + required_features & ~available_features; + iree_bitfield_string_temp_t temp0, temp1, temp2; + iree_string_view_t available_features_str = + iree_vm_bytecode_features_format(available_features, &temp0); + iree_string_view_t required_features_str = + iree_vm_bytecode_features_format(required_features, &temp1); + iree_string_view_t needed_features_str = + iree_vm_bytecode_features_format(needed_features, &temp2); + return iree_make_status_with_location( + file, line, IREE_STATUS_INVALID_ARGUMENT, + "required module features [%.*s] are not available in this runtime " + "configuration; have [%.*s] while module requires [%.*s]", + (int)needed_features_str.size, needed_features_str.data, + (int)available_features_str.size, available_features_str.data, + (int)required_features_str.size, required_features_str.data); +#else + return iree_status_from_code(IREE_STATUS_INVALID_ARGUMENT); +#endif // IREE_STATUS_MODE +}
diff --git a/runtime/src/iree/vm/bytecode/utils/features.h b/runtime/src/iree/vm/bytecode/utils/features.h new file mode 100644 index 0000000..6545803 --- /dev/null +++ b/runtime/src/iree/vm/bytecode/utils/features.h
@@ -0,0 +1,37 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_VM_BYTECODE_UTILS_FEATURES_H_ +#define IREE_VM_BYTECODE_UTILS_FEATURES_H_ + +#include "iree/base/api.h" +#include "iree/vm/api.h" +#include "iree/vm/bytecode/utils/isa.h" + +#ifdef __cplusplus +extern "C" { +#endif // __cplusplus + +// Formats a buffer usage bitfield as a string. +// See iree_bitfield_format for usage. +iree_string_view_t iree_vm_bytecode_features_format( + iree_vm_FeatureBits_enum_t value, iree_bitfield_string_temp_t* out_temp); + +// Returns the features available in this build of the runtime. +iree_vm_FeatureBits_enum_t iree_vm_bytecode_available_features(void); + +// Returns a pretty status reported at |file|/|line| when one or more features +// from |required_features| is missing from |available_features|. +// Returns OK if all required features are available. +iree_status_t iree_vm_check_feature_mismatch( + const char* file, int line, iree_vm_FeatureBits_enum_t required_features, + iree_vm_FeatureBits_enum_t available_features); + +#ifdef __cplusplus +} // extern "C" +#endif // __cplusplus + +#endif // IREE_VM_BYTECODE_UTILS_FEATURES_H_
diff --git a/runtime/src/iree/vm/bytecode/generated/.clang-format b/runtime/src/iree/vm/bytecode/utils/generated/.clang-format similarity index 100% rename from runtime/src/iree/vm/bytecode/generated/.clang-format rename to runtime/src/iree/vm/bytecode/utils/generated/.clang-format
diff --git a/runtime/src/iree/vm/bytecode/generated/op_table.h b/runtime/src/iree/vm/bytecode/utils/generated/op_table.h similarity index 100% rename from runtime/src/iree/vm/bytecode/generated/op_table.h rename to runtime/src/iree/vm/bytecode/utils/generated/op_table.h
diff --git a/runtime/src/iree/vm/bytecode/utils/isa.h b/runtime/src/iree/vm/bytecode/utils/isa.h new file mode 100644 index 0000000..d24300c --- /dev/null +++ b/runtime/src/iree/vm/bytecode/utils/isa.h
@@ -0,0 +1,103 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_VM_BYTECODE_UTILS_ISA_H_ +#define IREE_VM_BYTECODE_UTILS_ISA_H_ + +#include "iree/base/api.h" +#include "iree/vm/api.h" +#include "iree/vm/bytecode/utils/generated/op_table.h" // IWYU pragma: export + +// NOTE: include order matters: +#include "iree/base/internal/flatcc/parsing.h" // IWYU pragma: export +#include "iree/schemas/bytecode_module_def_reader.h" // IWYU pragma: export +#include "iree/schemas/bytecode_module_def_verifier.h" // IWYU pragma: export + +//===----------------------------------------------------------------------===// +// Misc utilities +//===----------------------------------------------------------------------===// + +#define VMMAX(a, b) (((a) > (b)) ? (a) : (b)) +#define VMMIN(a, b) (((a) < (b)) ? (a) : (b)) + +#define VM_AlignPC(pc, alignment) \ + (pc) = ((pc) + ((alignment)-1)) & ~((alignment)-1) + +//===----------------------------------------------------------------------===// +// Bytecode versioning +//===----------------------------------------------------------------------===// + +// Major bytecode version; mismatches on this will fail in either direction. +// This allows coarse versioning of completely incompatible versions. +// Matches BytecodeEncoder::kVersionMajor in the compiler. +#define IREE_VM_BYTECODE_VERSION_MAJOR 14 +// Minor bytecode version; lower versions are allowed to enable newer runtimes +// to load older serialized files when there are backwards-compatible changes. +// Higher versions are disallowed as they occur when new ops are added that +// otherwise cannot be executed by older runtimes. +// Matches BytecodeEncoder::kVersionMinor in the compiler. +#define IREE_VM_BYTECODE_VERSION_MINOR 0 + +//===----------------------------------------------------------------------===// +// Bytecode structural constants +//===----------------------------------------------------------------------===// + +// Size of a register ordinal in the bytecode. +#define IREE_REGISTER_ORDINAL_SIZE sizeof(uint16_t) + +// Maximum register count per bank. +// This determines the bits required to reference registers in the VM bytecode. +#define IREE_I32_REGISTER_COUNT 0x7FFF +#define IREE_REF_REGISTER_COUNT 0x3FFF + +#define IREE_I32_REGISTER_MASK 0x7FFF + +#define IREE_REF_REGISTER_TYPE_BIT 0x8000 +#define IREE_REF_REGISTER_MOVE_BIT 0x4000 +#define IREE_REF_REGISTER_MASK 0x3FFF + +// Maximum program counter offset within in a single block. +// This is just so that we can steal bits for flags and such. 16MB (today) +// should be more than enough for a single basic block. If not then we should +// compress better! +#define IREE_VM_PC_BLOCK_MAX 0x00FFFFFFu + +// Bytecode data -offset used when looking for the start of the currently +// dispatched instruction: `instruction_start = pc - OFFSET` +#define IREE_VM_PC_OFFSET_CORE 1 +#define IREE_VM_PC_OFFSET_EXT_I32 2 +#define IREE_VM_PC_OFFSET_EXT_F32 2 +#define IREE_VM_PC_OFFSET_EXT_F64 2 + +// Interleaved src-dst register sets for branch register remapping. +// This structure is an overlay for the bytecode that is serialized in a +// matching format. +typedef struct iree_vm_register_remap_list_t { + uint16_t size; + struct pair { + uint16_t src_reg; + uint16_t dst_reg; + } pairs[]; +} iree_vm_register_remap_list_t; +static_assert(iree_alignof(iree_vm_register_remap_list_t) == 2, + "Expecting byte alignment (to avoid padding)"); +static_assert(offsetof(iree_vm_register_remap_list_t, pairs) == 2, + "Expect no padding in the struct"); + +//===----------------------------------------------------------------------===// +// Bytecode data reading with little-/big-endian support +//===----------------------------------------------------------------------===// + +// Bytecode data access macros for reading values of a given type from a byte +// offset within the current function. +#define OP_I8(i) iree_unaligned_load_le((uint8_t*)&bytecode_data[pc + (i)]) +#define OP_I16(i) iree_unaligned_load_le((uint16_t*)&bytecode_data[pc + (i)]) +#define OP_I32(i) iree_unaligned_load_le((uint32_t*)&bytecode_data[pc + (i)]) +#define OP_I64(i) iree_unaligned_load_le((uint64_t*)&bytecode_data[pc + (i)]) +#define OP_F32(i) iree_unaligned_load_le((float*)&bytecode_data[pc + (i)]) +#define OP_F64(i) iree_unaligned_load_le((double*)&bytecode_data[pc + (i)]) + +#endif // IREE_VM_BYTECODE_UTILS_ISA_H_
diff --git a/runtime/src/iree/vm/bytecode/verifier.c b/runtime/src/iree/vm/bytecode/verifier.c new file mode 100644 index 0000000..f26c4f3 --- /dev/null +++ b/runtime/src/iree/vm/bytecode/verifier.c
@@ -0,0 +1,1863 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#include "iree/vm/bytecode/verifier.h" + +#include "iree/base/internal/math.h" +#include "iree/vm/bytecode/utils/block_list.h" +#include "iree/vm/bytecode/utils/features.h" + +//===----------------------------------------------------------------------===// +// Module metadata verification +//===----------------------------------------------------------------------===// + +iree_status_t iree_vm_bytecode_module_flatbuffer_verify( + iree_const_byte_span_t archive_contents, + iree_const_byte_span_t flatbuffer_contents, + iree_host_size_t archive_rodata_offset) { + // Run flatcc generated verification. This ensures all pointers are in-bounds + // and that we can safely walk the file, but not that the actual contents of + // the FlatBuffer meet our expectations. + int verify_ret = iree_vm_BytecodeModuleDef_verify_as_root( + flatbuffer_contents.data, flatbuffer_contents.data_length); + if (verify_ret != flatcc_verify_ok) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "FlatBuffer verification failed: %s", + flatcc_verify_error_string(verify_ret)); + } + + iree_vm_BytecodeModuleDef_table_t module_def = + iree_vm_BytecodeModuleDef_as_root(flatbuffer_contents.data); + + const iree_vm_FeatureBits_enum_t available_features = + iree_vm_bytecode_available_features(); + const iree_vm_FeatureBits_enum_t required_features = + iree_vm_BytecodeModuleDef_requirements(module_def); + IREE_RETURN_IF_ERROR(iree_vm_check_feature_mismatch( + __FILE__, __LINE__, required_features, available_features)); + + flatbuffers_string_t name = iree_vm_BytecodeModuleDef_name(module_def); + if (!flatbuffers_string_len(name)) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "module missing name field"); + } + + iree_vm_TypeDef_vec_t types = iree_vm_BytecodeModuleDef_types(module_def); + for (size_t i = 0; i < iree_vm_TypeDef_vec_len(types); ++i) { + iree_vm_TypeDef_table_t type_def = iree_vm_TypeDef_vec_at(types, i); + if (!type_def) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "types[%zu] missing body", i); + } + flatbuffers_string_t full_name = iree_vm_TypeDef_full_name(type_def); + if (flatbuffers_string_len(full_name) <= 0) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "types[%zu] missing name", i); + } + } + + iree_vm_RodataSegmentDef_vec_t rodata_segments = + iree_vm_BytecodeModuleDef_rodata_segments(module_def); + for (size_t i = 0; i < iree_vm_RodataSegmentDef_vec_len(rodata_segments); + ++i) { + iree_vm_RodataSegmentDef_table_t segment = + iree_vm_RodataSegmentDef_vec_at(rodata_segments, i); + if (iree_vm_RodataSegmentDef_embedded_data_is_present(segment)) { + continue; // embedded data is verified by FlatBuffers + } + uint64_t segment_offset = + iree_vm_RodataSegmentDef_external_data_offset(segment); + uint64_t segment_length = + iree_vm_RodataSegmentDef_external_data_length(segment); + uint64_t segment_end = + archive_rodata_offset + segment_offset + segment_length; + if (segment_end > archive_contents.data_length) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "rodata[%zu] external reference out of range", i); + } + } + + iree_vm_ModuleDependencyDef_vec_t dependencies = + iree_vm_BytecodeModuleDef_dependencies(module_def); + for (size_t i = 0; i < iree_vm_ModuleDependencyDef_vec_len(dependencies); + ++i) { + iree_vm_ModuleDependencyDef_table_t dependency_def = + iree_vm_ModuleDependencyDef_vec_at(dependencies, i); + flatbuffers_string_t module_name = + iree_vm_ModuleDependencyDef_name(dependency_def); + if (flatbuffers_string_len(module_name) == 0) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "dependencies[%zu] has no module name", i); + } + } + + iree_vm_ImportFunctionDef_vec_t imported_functions = + iree_vm_BytecodeModuleDef_imported_functions(module_def); + iree_vm_ExportFunctionDef_vec_t exported_functions = + iree_vm_BytecodeModuleDef_exported_functions(module_def); + iree_vm_FunctionSignatureDef_vec_t function_signatures = + iree_vm_BytecodeModuleDef_function_signatures(module_def); + iree_vm_FunctionDescriptor_vec_t function_descriptors = + iree_vm_BytecodeModuleDef_function_descriptors(module_def); + + for (size_t i = 0; + i < iree_vm_FunctionSignatureDef_vec_len(function_signatures); ++i) { + iree_vm_FunctionSignatureDef_table_t function_signature = + iree_vm_FunctionSignatureDef_vec_at(function_signatures, i); + if (!function_signature) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "function_signatures[%zu] missing body", i); + } + } + + for (size_t i = 0; i < iree_vm_ImportFunctionDef_vec_len(imported_functions); + ++i) { + iree_vm_ImportFunctionDef_table_t import_def = + iree_vm_ImportFunctionDef_vec_at(imported_functions, i); + if (!import_def) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "imports[%zu] missing body", i); + } + flatbuffers_string_t full_name = + iree_vm_ImportFunctionDef_full_name(import_def); + if (!flatbuffers_string_len(full_name)) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "imports[%zu] missing full_name", i); + } + } + + if (iree_vm_FunctionSignatureDef_vec_len(function_signatures) != + iree_vm_FunctionDescriptor_vec_len(function_descriptors)) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "function signature and descriptor table length mismatch (%zu vs %zu)", + iree_vm_FunctionSignatureDef_vec_len(function_signatures), + iree_vm_FunctionDescriptor_vec_len(function_descriptors)); + } + + for (size_t i = 0; i < iree_vm_ExportFunctionDef_vec_len(exported_functions); + ++i) { + iree_vm_ExportFunctionDef_table_t export_def = + iree_vm_ExportFunctionDef_vec_at(exported_functions, i); + if (!export_def) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "exports[%zu] missing body", i); + } + flatbuffers_string_t local_name = + iree_vm_ExportFunctionDef_local_name(export_def); + if (!flatbuffers_string_len(local_name)) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "exports[%zu] missing local_name", i); + } + iree_host_size_t internal_ordinal = + iree_vm_ExportFunctionDef_internal_ordinal(export_def); + if (internal_ordinal >= + iree_vm_FunctionDescriptor_vec_len(function_descriptors)) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "exports[%zu] internal_ordinal out of bounds (0 < %zu < %zu)", i, + internal_ordinal, + iree_vm_FunctionDescriptor_vec_len(function_descriptors)); + } + } + + // Verify that we can properly handle the bytecode embedded in the module. + // We require that major versions match and allow loading of older minor + // versions (we keep changes backwards-compatible). + const uint32_t bytecode_version = + iree_vm_BytecodeModuleDef_bytecode_version(module_def); + const uint32_t bytecode_version_major = bytecode_version >> 16; + const uint32_t bytecode_version_minor = bytecode_version & 0xFFFF; + if ((bytecode_version_major != IREE_VM_BYTECODE_VERSION_MAJOR) || + (bytecode_version_minor > IREE_VM_BYTECODE_VERSION_MINOR)) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "bytecode version mismatch; runtime supports %d.%d, module has %d.%d", + IREE_VM_BYTECODE_VERSION_MAJOR, IREE_VM_BYTECODE_VERSION_MINOR, + bytecode_version_major, bytecode_version_minor); + } + + flatbuffers_uint8_vec_t bytecode_data = + iree_vm_BytecodeModuleDef_bytecode_data(module_def); + for (size_t i = 0; + i < iree_vm_FunctionDescriptor_vec_len(function_descriptors); ++i) { + iree_vm_FunctionDescriptor_struct_t function_descriptor = + iree_vm_FunctionDescriptor_vec_at(function_descriptors, i); + if (function_descriptor->block_count == 0) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "functions[%zu] descriptor block count is 0; " + "functions must have at least 1 block, expected %d", + i, function_descriptor->block_count); + } + if (function_descriptor->bytecode_length == 0) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "functions[%zu] descriptor bytecode reports 0 " + "length; functions must have at least one block", + i); + } + if (function_descriptor->bytecode_offset < 0 || + function_descriptor->bytecode_offset + + function_descriptor->bytecode_length > + flatbuffers_uint8_vec_len(bytecode_data)) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "functions[%zu] descriptor bytecode span out of " + "range (0 < %d < %" PRIhsz ")", + i, function_descriptor->bytecode_offset, + flatbuffers_uint8_vec_len(bytecode_data)); + } + if (function_descriptor->i32_register_count > IREE_I32_REGISTER_COUNT || + function_descriptor->ref_register_count > IREE_REF_REGISTER_COUNT) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "functions[%zu] descriptor register count out of range", i); + } + } + + return iree_ok_status(); +} + +//===----------------------------------------------------------------------===// +// Function verification +//===----------------------------------------------------------------------===// + +// State used during verification of a function. +typedef struct iree_vm_bytecode_verify_state_t { + // Within a block (encountered a block marker and not yet a terminator). + uint32_t in_block : 1; + + // Maximum valid register ordinals. + uint32_t i32_register_count; + uint32_t ref_register_count; + + // Parsed argument and result cconv fragments. + iree_string_view_t cconv_arguments; + iree_string_view_t cconv_results; + + // All block branch points. + iree_vm_bytecode_block_list_t block_list; + + // Quick lookups of flatbuffer properties. + const iree_vm_ImportFunctionDef_vec_t imported_functions; + const iree_vm_ExportFunctionDef_vec_t exported_functions; + const iree_vm_FunctionSignatureDef_vec_t function_signatures; + const iree_vm_FunctionDescriptor_vec_t function_descriptors; + iree_host_size_t rodata_storage_size; + iree_host_size_t rodata_ref_count; + iree_host_size_t rwdata_storage_size; + iree_host_size_t global_ref_count; +} iree_vm_bytecode_verify_state_t; + +// Parses the cconv fragments from the given |signature_def|. +static iree_status_t iree_vm_bytecode_function_get_cconv_fragments( + iree_vm_FunctionSignatureDef_table_t signature_def, + iree_string_view_t* out_arguments, iree_string_view_t* out_results); + +// Verifies that the function has storage for the declared arguments. +static iree_status_t iree_vm_bytecode_function_verify_arguments( + const iree_vm_bytecode_verify_state_t* verify_state); + +// Verifies a single operation at |pc| in the function |bytecode_data|. +// Returns an error if the op is invalid and otherwise sets |out_next_pc| to the +// program counter immediately following the op (which may be the end of data!). +static iree_status_t iree_vm_bytecode_function_verify_bytecode_op( + iree_vm_bytecode_module_t* module, + iree_vm_bytecode_verify_state_t* verify_state, + iree_vm_FunctionSignatureDef_table_t function_signature, + iree_vm_FunctionDescriptor_struct_t function_descriptor, + iree_const_byte_span_t bytecode_data, uint32_t pc, uint32_t max_pc, + uint32_t* out_next_pc); + +// NOTE: by the time this is called we have the module created and can assume +// all information on it has been verified. The only thing this verifies is +// function bytecode and capabilities! +iree_status_t iree_vm_bytecode_function_verify( + iree_vm_bytecode_module_t* module, uint16_t function_ordinal, + iree_allocator_t scratch_allocator) { + if (function_ordinal >= module->function_descriptor_count) { + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, + "invalid function ordinal"); + } + iree_vm_FunctionSignatureDef_table_t function_signature_def = + iree_vm_FunctionSignatureDef_vec_at( + iree_vm_BytecodeModuleDef_function_signatures(module->def), + function_ordinal); + const iree_vm_FunctionDescriptor_t* function_descriptor = + &module->function_descriptor_table[function_ordinal]; + + const iree_vm_FeatureBits_enum_t available_features = + iree_vm_bytecode_available_features(); + const iree_vm_FeatureBits_enum_t required_features = + function_descriptor->requirements; + IREE_RETURN_IF_ERROR(iree_vm_check_feature_mismatch( + __FILE__, __LINE__, required_features, available_features)); + + if (function_descriptor->block_count == 0) { + return iree_make_status( + IREE_STATUS_OUT_OF_RANGE, + "no blocks defined; functions must have at least one block"); + } + + // State used during verification. + iree_vm_bytecode_verify_state_t verify_state = { + .in_block = 0, + .imported_functions = + iree_vm_BytecodeModuleDef_imported_functions(module->def), + .exported_functions = + iree_vm_BytecodeModuleDef_exported_functions(module->def), + .function_signatures = + iree_vm_BytecodeModuleDef_function_signatures(module->def), + .function_descriptors = + iree_vm_BytecodeModuleDef_function_descriptors(module->def), + .rodata_storage_size = 0, + .rodata_ref_count = 0, + .rwdata_storage_size = 0, + .global_ref_count = 0, + }; + + // NOTE: these must be consistent with iree_vm_bytecode_module_layout_state. + verify_state.rodata_storage_size = 0; + verify_state.rodata_ref_count = iree_vm_RodataSegmentDef_vec_len( + iree_vm_BytecodeModuleDef_rodata_segments(module->def)); + iree_vm_ModuleStateDef_table_t module_state_def = + iree_vm_BytecodeModuleDef_module_state(module->def); + if (module_state_def) { + verify_state.rwdata_storage_size = + iree_vm_ModuleStateDef_global_bytes_capacity(module_state_def); + verify_state.global_ref_count = + iree_vm_ModuleStateDef_global_ref_count(module_state_def); + } + + // Ensure the register storage (rounded to the nearest power of 2) won't + // exceed the maximum allowed registers. + verify_state.i32_register_count = iree_math_round_up_to_pow2_u32( + VMMAX(1, function_descriptor->i32_register_count)); + verify_state.ref_register_count = iree_math_round_up_to_pow2_u32( + VMMAX(1, function_descriptor->ref_register_count)); + if (IREE_UNLIKELY(verify_state.i32_register_count > IREE_I32_REGISTER_MASK) || + IREE_UNLIKELY(verify_state.ref_register_count > IREE_REF_REGISTER_MASK)) { + // Register count overflow. A valid compiler should never produce files that + // hit this. + return iree_make_status(IREE_STATUS_RESOURCE_EXHAUSTED, + "register count overflow"); + } + + // Grab the cconv fragments declaring the arguments/results of the function. + verify_state.cconv_arguments = iree_string_view_empty(); + verify_state.cconv_results = iree_string_view_empty(); + IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_get_cconv_fragments( + function_signature_def, &verify_state.cconv_arguments, + &verify_state.cconv_results)); + + // Verify there is storage for passed arguments. + IREE_RETURN_IF_ERROR( + iree_vm_bytecode_function_verify_arguments(&verify_state)); + + // NOTE: module verification ensures the function descriptor has a valid + // bytecode range so we can assume that's true here. + IREE_ASSERT(function_descriptor->bytecode_length > 0); + IREE_ASSERT(function_descriptor->bytecode_offset >= 0); + IREE_ASSERT(function_descriptor->bytecode_offset + + function_descriptor->bytecode_length <= + module->bytecode_data.data_length); + iree_const_byte_span_t bytecode_data = iree_make_const_byte_span( + module->bytecode_data.data + function_descriptor->bytecode_offset, + function_descriptor->bytecode_length); + const uint32_t max_pc = (uint32_t)function_descriptor->bytecode_length; + + // Reserve the block list. As we walk the bytecode we'll declare/define blocks + // and then afterward verify all were found. + IREE_ASSERT(function_descriptor->block_count > 0); + IREE_RETURN_IF_ERROR(iree_vm_bytecode_block_list_initialize( + function_descriptor->block_count, scratch_allocator, + &verify_state.block_list)); + + // Perform bytecode verification by performing a single-pass walk of all + // function bytecode. + iree_status_t status = iree_ok_status(); + for (uint32_t pc = 0; pc < bytecode_data.data_length - 1;) { + uint32_t start_pc = pc; + status = iree_vm_bytecode_function_verify_bytecode_op( + module, &verify_state, function_signature_def, function_descriptor, + bytecode_data, start_pc, max_pc, &pc); + if (!iree_status_is_ok(status)) { +#if IREE_STATUS_MODE + // To get a useful source location we have to ask the main module; the + // base function table may only contain public symbols and not any + // internal ones. + iree_string_view_t module_name = iree_vm_module_name(&module->interface); + iree_vm_function_t function = {0}; + iree_status_ignore(iree_vm_module_lookup_function_by_ordinal( + &module->interface, IREE_VM_FUNCTION_LINKAGE_INTERNAL, + function_ordinal, &function)); + iree_string_view_t function_name = iree_vm_function_name(&function); + if (!iree_string_view_is_empty(function_name)) { + status = iree_status_annotate_f(status, "at %.*s.%.*s+%08X", + (int)module_name.size, module_name.data, + (int)function_name.size, + function_name.data, start_pc); + } else { + status = iree_status_annotate_f(status, "at %.*s@%u+%08X", + (int)module_name.size, module_name.data, + function_ordinal, start_pc); + } +#endif // IREE_STATUS_MODE + break; + } + } + + // Ensure there was a terminator. + if (iree_status_is_ok(status) && verify_state.in_block) { + status = iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "function missing terminator in the last block"); + } + + // Verify all blocks are defined and have proper markers. + if (iree_status_is_ok(status)) { + status = iree_vm_bytecode_block_list_verify(&verify_state.block_list, + bytecode_data); + } + + iree_vm_bytecode_block_list_deinitialize(&verify_state.block_list, + scratch_allocator); + + return status; +} + +//===----------------------------------------------------------------------===// +// Utilities matching the tablegen op encoding scheme +//===----------------------------------------------------------------------===// +// These utilities match the VM_Enc* statements in VMBase.td 1:1, allowing us +// to have the inverse of the encoding which make things easier to read. +// +// Each macro will increment the pc by the number of bytes read and as such must +// be called in the same order the values are encoded. + +// Bails if the |pc| exceeds the |max_pc|. +#define IREE_VM_VERIFY_PC_RANGE(pc, max_pc) \ + if (IREE_UNLIKELY((pc) > (max_pc))) { \ + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ + "bytecode data overrun trying to parsing op at " \ + "%08X (%u) of %u available bytes", \ + (uint32_t)(pc), (uint32_t)(pc), \ + (uint32_t)(max_pc)); \ + } + +// Bails if the function doesn't have the given |required_features| declared. +#define IREE_VM_VERIFY_REQUIREMENT(required_features) \ + if (IREE_UNLIKELY(!iree_all_bits_set(function_descriptor->requirements, \ + (required_features)))) { \ + return iree_vm_check_feature_mismatch(__FILE__, __LINE__, \ + (required_features), \ + function_descriptor->requirements); \ + } + +// Bails if the register ordinal for the given register type is out of bounds. +#define IREE_VM_VERIFY_REG_ORDINAL(name) \ + IREE_VM_VERIFY_PC_RANGE(pc + IREE_REGISTER_ORDINAL_SIZE, max_pc); \ + const uint32_t name = OP_I16(0); +#define IREE_VM_VERIFY_REG_ORDINAL_X32(ordinal, category) \ + if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_TYPE_BIT) != 0)) { \ + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ + category \ + " register required but ref register %u provided", \ + (ordinal)); \ + } else if (IREE_UNLIKELY((ordinal) >= verify_state->i32_register_count)) { \ + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ + category " register ordinal %u out of range %u", \ + (ordinal), verify_state->i32_register_count); \ + } +#define IREE_VM_VERIFY_REG_ORDINAL_X64(ordinal, category) \ + if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_TYPE_BIT) != 0)) { \ + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ + category \ + " register required but ref register %u provided", \ + (ordinal)); \ + } else if (IREE_UNLIKELY((ordinal & 1) != 0)) { \ + return iree_make_status( \ + IREE_STATUS_INVALID_ARGUMENT, \ + category " register ordinal %u not 8-byte aligned", (ordinal)); \ + } else if (IREE_UNLIKELY((ordinal) + 1 >= \ + verify_state->i32_register_count)) { \ + return iree_make_status( \ + IREE_STATUS_OUT_OF_RANGE, \ + category " register ordinal %u:%u out of range %u", (ordinal), \ + (ordinal) + 1, verify_state->i32_register_count); \ + } +#define IREE_VM_VERIFY_REG_I32(ordinal) \ + IREE_VM_VERIFY_REG_ORDINAL_X32(ordinal, "i32"); +#define IREE_VM_VERIFY_REG_I64(ordinal) \ + IREE_VM_VERIFY_REG_ORDINAL_X64(ordinal, "i64"); +#define IREE_VM_VERIFY_REG_F32(ordinal) \ + IREE_VM_VERIFY_REG_ORDINAL_X32(ordinal, "f32"); +#define IREE_VM_VERIFY_REG_F64(ordinal) \ + IREE_VM_VERIFY_REG_ORDINAL_X64(ordinal, "f64"); +#define IREE_VM_VERIFY_REG_REF(ordinal) \ + if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_TYPE_BIT) == 0)) { \ + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ + "ref register required but non-ref %u provided", \ + (ordinal)); \ + } else if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_MASK) >= \ + verify_state->ref_register_count)) { \ + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ + "ref register ordinal %u out of range %u", \ + (ordinal), verify_state->ref_register_count); \ + } +#define IREE_VM_VERIFY_REG_ANY(ordinal) \ + if (IREE_UNLIKELY(((ordinal)&IREE_REF_REGISTER_TYPE_BIT) == 0)) { \ + } else { \ + } + +#define VM_VerifyConstI8(name) \ + IREE_VM_VERIFY_PC_RANGE(pc + 1, max_pc); \ + uint8_t name = OP_I8(0); \ + (void)(name); \ + ++pc; +#define VM_VerifyConstI32(name) \ + IREE_VM_VERIFY_PC_RANGE(pc + 4, max_pc); \ + uint32_t name = OP_I32(0); \ + (void)(name); \ + pc += 4; +#define VM_VerifyConstI64(name) \ + IREE_VM_VERIFY_PC_RANGE(pc + 8, max_pc); \ + uint32_t name = OP_I64(0); \ + (void)(name); \ + pc += 8; +#define VM_VerifyConstF32(name) \ + IREE_VM_VERIFY_PC_RANGE(pc + 4, max_pc); \ + uint32_t name = OP_F32(0); \ + (void)(name); \ + pc += 4; +#define VM_VerifyConstF64(name) \ + IREE_VM_VERIFY_PC_RANGE(pc + 8, max_pc); \ + uint32_t name = OP_F64(0); \ + (void)(name); \ + pc += 8; + +#define VM_VerifyFuncAttr(name) VM_VerifyConstI32(name) +#define VM_IsImportOrdinal(name) (((name)&0x80000000u) != 0) +#define VM_UnmaskImportOrdinal(name) name &= ~0x80000000u +#define VM_VerifyImportOrdinal(name) \ + if (IREE_UNLIKELY((name) >= iree_vm_ImportFunctionDef_vec_len( \ + verify_state->imported_functions))) { \ + return iree_make_status( \ + IREE_STATUS_INVALID_ARGUMENT, "import ordinal %u out of range %zu", \ + name, \ + iree_vm_ImportFunctionDef_vec_len(verify_state->imported_functions)); \ + } +#define VM_VerifyFunctionOrdinal(name) \ + if (IREE_UNLIKELY((name)) >= iree_vm_FunctionDescriptor_vec_len( \ + verify_state->function_descriptors)) { \ + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ + "function ordinal %u out of range %zu", (name), \ + iree_vm_FunctionDescriptor_vec_len( \ + verify_state->function_descriptors)); \ + } +#define VM_VerifyGlobalAttr(name) VM_VerifyConstI32(name) +#define VM_VerifyRwdataOffset(name, access_length) \ + if (IREE_UNLIKELY(((name) + (access_length)) > \ + verify_state->rwdata_storage_size)) { \ + return iree_make_status( \ + IREE_STATUS_OUT_OF_RANGE, \ + "global byte_offset out of range: %d (rwdata=%zu)", (name), \ + verify_state->rwdata_storage_size); \ + } +#define VM_VerifyGlobalRefOrdinal(name) \ + if (IREE_UNLIKELY((name) >= verify_state->global_ref_count)) { \ + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ + "global ref ordinal out of range: %d (table=%zu)", \ + (name), verify_state->global_ref_count); \ + } +#define VM_VerifyRodataAttr(name) VM_VerifyConstI32(name) +#define VM_VerifyRodataOrdinal(name) \ + if (IREE_UNLIKELY((name) >= verify_state->rodata_ref_count)) { \ + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ + "rodata ref ordinal out of range: %d (table=%zu)", \ + (name), verify_state->rodata_ref_count); \ + } +#define VM_VerifyType(name) \ + IREE_VM_VERIFY_PC_RANGE(pc + 4, max_pc); \ + uint32_t name##_id = OP_I32(0); \ + if (IREE_UNLIKELY(name##_id >= module->type_count)) { \ + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ + "type id ordinal out of range: %d (table=%zu)", \ + name##_id, module->type_count); \ + } \ + const iree_vm_type_def_t* name = &module->type_table[name##_id]; \ + (void)(name); \ + pc += 4; +#define VM_VerifyTypeOf(name) VM_VerifyType(name) +#define VM_VerifyIntAttr32(name) VM_VerifyConstI32(name) +#define VM_VerifyIntAttr64(name) VM_VerifyConstI64(name) +#define VM_VerifyFloatAttr32(name) VM_VerifyConstF32(name) +#define VM_VerifyFloatAttr64(name) VM_VerifyConstF64(name) +#define VM_VerifyStrAttr(name, out_str) \ + IREE_VM_VERIFY_PC_RANGE(pc + 2, max_pc); \ + (out_str)->size = (iree_host_size_t)OP_I16(0); \ + IREE_VM_VERIFY_PC_RANGE(pc + 2 + (out_str)->size, max_pc); \ + (out_str)->data = (const char*)&bytecode_data[pc + 2]; \ + pc += 2 + (out_str)->size; + +#define VM_VerifyBranchTarget(name) \ + VM_VerifyConstI32(name##_pc); \ + iree_vm_bytecode_block_t* name = NULL; \ + IREE_RETURN_IF_ERROR(iree_vm_bytecode_block_list_insert( \ + &verify_state->block_list, name##_pc, &name)); +#define VM_VerifyBranchOperands(name) \ + VM_AlignPC(pc, IREE_REGISTER_ORDINAL_SIZE); \ + IREE_VM_VERIFY_PC_RANGE(pc + IREE_REGISTER_ORDINAL_SIZE, max_pc); \ + const iree_vm_register_remap_list_t* name = \ + (const iree_vm_register_remap_list_t*)&bytecode_data[pc]; \ + pc += IREE_REGISTER_ORDINAL_SIZE; \ + IREE_VM_VERIFY_PC_RANGE(pc + (name)->size * 2 * IREE_REGISTER_ORDINAL_SIZE, \ + max_pc); \ + pc += (name)->size * 2 * IREE_REGISTER_ORDINAL_SIZE; \ + for (uint16_t i = 0; i < name->size; ++i) { \ + IREE_VM_VERIFY_REG_ANY(name->pairs[i].src_reg); \ + IREE_VM_VERIFY_REG_ANY(name->pairs[i].dst_reg); \ + } + +#define VM_VerifyOperandRegI32(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_I32(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyOperandRegI64(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_I64(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyOperandRegI64HostSize(name) VM_VerifyOperandRegI64(name) +#define VM_VerifyOperandRegF32(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_F32(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyOperandRegF64(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_F32(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyOperandRegRef(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_REF(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyVariadicOperands(name) \ + VM_AlignPC(pc, IREE_REGISTER_ORDINAL_SIZE); \ + IREE_VM_VERIFY_PC_RANGE(pc + IREE_REGISTER_ORDINAL_SIZE, max_pc); \ + const iree_vm_register_list_t* name = \ + (const iree_vm_register_list_t*)&bytecode_data[pc]; \ + pc += IREE_REGISTER_ORDINAL_SIZE; \ + IREE_VM_VERIFY_PC_RANGE(pc + (name)->size * IREE_REGISTER_ORDINAL_SIZE, \ + max_pc); \ + pc += (name)->size * IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyVariadicOperandsI32(name) \ + VM_VerifyVariadicOperands(name); \ + for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ + IREE_VM_VERIFY_REG_I32((name)->registers[__i]); \ + } +#define VM_VerifyVariadicOperandsI64(name) \ + VM_VerifyVariadicOperands(name); \ + for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ + IREE_VM_VERIFY_REG_I64((name)->registers[__i]); \ + } +#define VM_VerifyVariadicOperandsF32(name) \ + VM_VerifyVariadicOperands(name); \ + for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ + IREE_VM_VERIFY_REG_F32((name)->registers[__i]); \ + } +#define VM_VerifyVariadicOperandsF64(name) \ + VM_VerifyVariadicOperands(name); \ + for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ + IREE_VM_VERIFY_REG_F64((name)->registers[__i]); \ + } +#define VM_VerifyVariadicOperandsRef(name, type_def) \ + VM_VerifyVariadicOperands(name); \ + for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ + IREE_VM_VERIFY_REG_REF((name)->registers[__i]); \ + } +#define VM_VerifyVariadicOperandsAny(name) \ + VM_VerifyVariadicOperands(name); \ + for (uint16_t __i = 0; __i < (name)->size; ++__i) { \ + IREE_VM_VERIFY_REG_ANY((name)->registers[__i]); \ + } +#define VM_VerifyResultRegI32(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_I32(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyResultRegI64(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_I64(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyResultRegF32(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_F32(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyResultRegF64(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_F64(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyResultRegRef(name) \ + IREE_VM_VERIFY_REG_ORDINAL(name##_ordinal); \ + IREE_VM_VERIFY_REG_REF(name##_ordinal); \ + pc += IREE_REGISTER_ORDINAL_SIZE; +#define VM_VerifyVariadicResultsAny(name) VM_VerifyVariadicOperandsAny(name) + +#define VERIFY_OP_CORE_UNARY_I32(op_name) \ + VERIFY_OP(CORE, op_name, { \ + VM_VerifyOperandRegI32(operand); \ + VM_VerifyResultRegI32(result); \ + }); + +#define VERIFY_OP_CORE_UNARY_I64(op_name) \ + VERIFY_OP(CORE, op_name, { \ + VM_VerifyOperandRegI64(operand); \ + VM_VerifyResultRegI64(result); \ + }); + +#define VERIFY_OP_CORE_BINARY_I32(op_name) \ + VERIFY_OP(CORE, op_name, { \ + VM_VerifyOperandRegI32(lhs); \ + VM_VerifyOperandRegI32(rhs); \ + VM_VerifyResultRegI32(result); \ + }); + +#define VERIFY_OP_CORE_BINARY_I64(op_name) \ + VERIFY_OP(CORE, op_name, { \ + VM_VerifyOperandRegI64(lhs); \ + VM_VerifyOperandRegI64(rhs); \ + VM_VerifyResultRegI64(result); \ + }); + +#define VERIFY_OP_CORE_TERNARY_I32(op_name) \ + VERIFY_OP(CORE, op_name, { \ + VM_VerifyOperandRegI32(a); \ + VM_VerifyOperandRegI32(b); \ + VM_VerifyOperandRegI32(c); \ + VM_VerifyResultRegI32(result); \ + }); + +#define VERIFY_OP_CORE_TERNARY_I64(op_name) \ + VERIFY_OP(CORE, op_name, { \ + VM_VerifyOperandRegI64(a); \ + VM_VerifyOperandRegI64(b); \ + VM_VerifyOperandRegI64(c); \ + VM_VerifyResultRegI64(result); \ + }); + +#define VERIFY_OP_EXT_F32_UNARY_F32(op_name) \ + VERIFY_OP(EXT_F32, op_name, { \ + VM_VerifyOperandRegF32(operand); \ + VM_VerifyResultRegF32(result); \ + }); + +#define VERIFY_OP_EXT_F32_BINARY_F32(op_name) \ + VERIFY_OP(EXT_F32, op_name, { \ + VM_VerifyOperandRegF32(lhs); \ + VM_VerifyOperandRegF32(rhs); \ + VM_VerifyResultRegF32(result); \ + }); + +#define VERIFY_OP_EXT_F32_TERNARY_F32(op_name) \ + VERIFY_OP(EXT_F32, op_name, { \ + VM_VerifyOperandRegF32(a); \ + VM_VerifyOperandRegF32(b); \ + VM_VerifyOperandRegF32(c); \ + VM_VerifyResultRegF32(result); \ + }); + +#define VERIFY_OP_EXT_F64_UNARY_F64(op_name) \ + VERIFY_OP(EXT_F64, op_name, { \ + VM_VerifyOperandRegF64(operand); \ + VM_VerifyResultRegF64(result); \ + }); + +#define VERIFY_OP_EXT_F64_BINARY_F64(op_name) \ + VERIFY_OP(EXT_F64, op_name, { \ + VM_VerifyOperandRegF64(lhs); \ + VM_VerifyOperandRegF64(rhs); \ + VM_VerifyResultRegF64(result); \ + }); + +#define VERIFY_OP_EXT_F64_TERNARY_F64(op_name) \ + VERIFY_OP(EXT_F64, op_name, { \ + VM_VerifyOperandRegF64(a); \ + VM_VerifyOperandRegF64(b); \ + VM_VerifyOperandRegF64(c); \ + VM_VerifyResultRegF64(result); \ + }); + +//===----------------------------------------------------------------------===// +// Call verification +//===----------------------------------------------------------------------===// + +static iree_status_t iree_vm_bytecode_function_get_cconv_fragments( + iree_vm_FunctionSignatureDef_table_t signature_def, + iree_string_view_t* out_arguments, iree_string_view_t* out_results) { + flatbuffers_string_t cconv_str = + iree_vm_FunctionSignatureDef_calling_convention(signature_def); + iree_vm_function_signature_t signature = { + .calling_convention = + iree_make_string_view(cconv_str, flatbuffers_string_len(cconv_str)), + }; + return iree_vm_function_call_get_cconv_fragments(&signature, out_arguments, + out_results); +} + +static iree_status_t iree_vm_bytecode_function_count_cconv_regs( + iree_string_view_t cconv_fragment, iree_host_size_t* out_i32_count, + iree_host_size_t* out_ref_count) { + iree_host_size_t i32_count = 0; + iree_host_size_t ref_count = 0; + for (iree_host_size_t i = 0; i < cconv_fragment.size; ++i) { + switch (cconv_fragment.data[i]) { + default: + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "unsupported cconv fragment char '%c'", + cconv_fragment.data[i]); + case IREE_VM_CCONV_TYPE_VOID: + break; + case IREE_VM_CCONV_TYPE_I32: + case IREE_VM_CCONV_TYPE_F32: + ++i32_count; + break; + case IREE_VM_CCONV_TYPE_I64: + case IREE_VM_CCONV_TYPE_F64: + if ((i32_count % 2) != 0) { + // Unaligned; pad an i32 register to get to i64 alignment. + ++i32_count; + } + i32_count += 2; + break; + case IREE_VM_CCONV_TYPE_REF: + ++ref_count; + break; + case IREE_VM_CCONV_TYPE_SPAN_START: + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "internal functions cannot accept variadic arguments"); + } + } + *out_i32_count = i32_count; + *out_ref_count = ref_count; + return iree_ok_status(); +} + +static iree_status_t iree_vm_bytecode_function_verify_arguments( + const iree_vm_bytecode_verify_state_t* verify_state) { + iree_host_size_t args_i32 = 0; + iree_host_size_t args_ref = 0; + IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_count_cconv_regs( + verify_state->cconv_arguments, &args_i32, &args_ref)); + iree_host_size_t rets_i32 = 0; + iree_host_size_t rets_ref = 0; + IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_count_cconv_regs( + verify_state->cconv_results, &rets_i32, &rets_ref)); + if (verify_state->i32_register_count < args_i32 || + verify_state->i32_register_count < rets_i32 || + verify_state->ref_register_count < args_ref || + verify_state->ref_register_count < rets_ref) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "insufficient register storage for function arguments/results"); + } + return iree_ok_status(); +} + +static iree_status_t iree_vm_bytecode_function_verify_cconv_register( + const iree_vm_bytecode_verify_state_t* verify_state, char cconv_type, + const iree_vm_register_list_t* IREE_RESTRICT reg_list, int reg_i) { + if (reg_i >= reg_list->size) { + return iree_make_status( + IREE_STATUS_OUT_OF_RANGE, + "register list underflow (have %u, trying to access %u)", + reg_list->size, reg_i); + } + switch (cconv_type) { + case IREE_VM_CCONV_TYPE_I32: + case IREE_VM_CCONV_TYPE_F32: { + IREE_VM_VERIFY_REG_ORDINAL_X32(reg_list->registers[reg_i], "i32/f32"); + } break; + case IREE_VM_CCONV_TYPE_I64: + case IREE_VM_CCONV_TYPE_F64: { + IREE_VM_VERIFY_REG_ORDINAL_X64(reg_list->registers[reg_i], "i64/f64"); + } break; + case IREE_VM_CCONV_TYPE_REF: { + IREE_VM_VERIFY_REG_REF(reg_list->registers[reg_i]); + } break; + default: + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "unsupported cconv fragment char '%c'", + cconv_type); + } + return iree_ok_status(); +} + +static iree_status_t iree_vm_bytecode_function_verify_cconv_registers( + const iree_vm_bytecode_verify_state_t* verify_state, + iree_string_view_t cconv_fragment, + const iree_vm_register_list_t* IREE_RESTRICT segment_size_list, + const iree_vm_register_list_t* IREE_RESTRICT reg_list) { + for (uint16_t i = 0, seg_i = 0, reg_i = 0; i < cconv_fragment.size; + ++i, ++seg_i) { + switch (cconv_fragment.data[i]) { + case IREE_VM_CCONV_TYPE_VOID: + break; + case IREE_VM_CCONV_TYPE_I32: + case IREE_VM_CCONV_TYPE_F32: + case IREE_VM_CCONV_TYPE_I64: + case IREE_VM_CCONV_TYPE_F64: + case IREE_VM_CCONV_TYPE_REF: { + IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_verify_cconv_register( + verify_state, cconv_fragment.data[i], reg_list, reg_i++)); + } break; + case IREE_VM_CCONV_TYPE_SPAN_START: { + if (!segment_size_list) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "function is variadic but no segment size list provided"); + } else if (seg_i >= segment_size_list->size) { + return iree_make_status( + IREE_STATUS_OUT_OF_RANGE, + "segment size list underflow (have %u, trying to access %u)", + segment_size_list->size, seg_i); + } + uint16_t span_count = segment_size_list->registers[seg_i]; + if (!span_count) { + // No items; skip the span. + do { + ++i; + } while (i < cconv_fragment.size && + cconv_fragment.data[i] != IREE_VM_CCONV_TYPE_SPAN_END); + continue; + } + uint16_t span_start_i = i + 1; + for (uint16_t j = 0; j < span_count; ++j) { + for (i = span_start_i; + i < cconv_fragment.size && + cconv_fragment.data[i] != IREE_VM_CCONV_TYPE_SPAN_END; + ++i) { + // TODO(benvanik): share with switch above. + switch (cconv_fragment.data[i]) { + case IREE_VM_CCONV_TYPE_VOID: + break; + case IREE_VM_CCONV_TYPE_I32: + case IREE_VM_CCONV_TYPE_F32: + case IREE_VM_CCONV_TYPE_I64: + case IREE_VM_CCONV_TYPE_F64: + case IREE_VM_CCONV_TYPE_REF: { + IREE_RETURN_IF_ERROR( + iree_vm_bytecode_function_verify_cconv_register( + verify_state, cconv_fragment.data[i], reg_list, + reg_i++)); + } break; + default: + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "unsupported cconv fragment char '%c'", + cconv_fragment.data[i]); + } + } + } + } break; + default: + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "unsupported cconv fragment char '%c'", + cconv_fragment.data[i]); + } + } + return iree_ok_status(); +} + +static iree_status_t iree_vm_bytecode_function_verify_call( + const iree_vm_bytecode_verify_state_t* verify_state, + iree_vm_FunctionSignatureDef_table_t signature_def, + const iree_vm_register_list_t* IREE_RESTRICT segment_size_list, + const iree_vm_register_list_t* IREE_RESTRICT src_reg_list, + const iree_vm_register_list_t* IREE_RESTRICT dst_reg_list) { + iree_string_view_t arguments = iree_string_view_empty(); + iree_string_view_t results = iree_string_view_empty(); + IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_get_cconv_fragments( + signature_def, &arguments, &results)); + IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_verify_cconv_registers( + verify_state, arguments, segment_size_list, src_reg_list)); + IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_verify_cconv_registers( + verify_state, results, /*segment_sizes=*/NULL, dst_reg_list)); + return iree_ok_status(); +} + +//===----------------------------------------------------------------------===// +// Bytecode verification +//===----------------------------------------------------------------------===// + +#define VERIFY_OP(ext, op_name, body) \ + case IREE_VM_OP_##ext##_##op_name: { \ + body; \ + } break; + +#define BEGIN_VERIFY_PREFIX(op_name, ext) \ + case IREE_VM_OP_CORE_##op_name: { \ + IREE_VM_VERIFY_PC_RANGE(pc + 1, max_pc); \ + IREE_VM_VERIFY_REQUIREMENT(ext); \ + switch (bytecode_data[pc++]) { +#define END_VERIFY_PREFIX() \ + default: \ + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, \ + "unhandled ext opcode"); \ + } \ + break; \ + } +#define UNHANDLED_VERIFY_PREFIX(op_name, ext) \ + case IREE_VM_OP_CORE_##op_name: { \ + return iree_vm_check_feature_mismatch(__FILE__, __LINE__, ext, \ + function_descriptor->requirements); \ + } + +static iree_status_t iree_vm_bytecode_function_verify_bytecode_op( + iree_vm_bytecode_module_t* module, + iree_vm_bytecode_verify_state_t* verify_state, + iree_vm_FunctionSignatureDef_table_t function_signature, + iree_vm_FunctionDescriptor_struct_t function_descriptor, + iree_const_byte_span_t function_bytecode, uint32_t start_pc, + uint32_t max_pc, uint32_t* out_next_pc) { + *out_next_pc = 0; + uint32_t pc = start_pc; + const uint8_t* bytecode_data = function_bytecode.data; + + // NOTE: we keep this as simple as possible so that we can one day auto + // generate it from tblgen, which has all the encodings in a similar form + // such that we could do string substitution to get the verifier macros. + + // All ops except for Block must be inside of a block. We hoist that check + // that here before switching out. + IREE_VM_VERIFY_PC_RANGE(pc + 1, max_pc); + if (verify_state->in_block == 0) { + // If not in a block then the next opcode must be a block. + if (bytecode_data[pc] != IREE_VM_OP_CORE_Block) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "op at pc %08X is not in a block", pc); + } + } else { + // If in a block then the next opcode must not be a block. + if (bytecode_data[pc] == IREE_VM_OP_CORE_Block) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "op at pc %08X is a block while still in a block", + pc); + } + } + + // Get primary opcode. All ops have at least 1 byte. + switch (bytecode_data[pc++]) { + //===------------------------------------------------------------------===// + // Globals + //===------------------------------------------------------------------===// + + VERIFY_OP(CORE, GlobalLoadI32, { + VM_VerifyGlobalAttr(byte_offset); + VM_VerifyRwdataOffset(byte_offset, 4); + VM_VerifyResultRegI32(value); + }); + + VERIFY_OP(CORE, GlobalStoreI32, { + VM_VerifyGlobalAttr(byte_offset); + VM_VerifyRwdataOffset(byte_offset, 4); + VM_VerifyOperandRegI32(value); + }); + + VERIFY_OP(CORE, GlobalLoadIndirectI32, { + VM_VerifyOperandRegI32(byte_offset); + // NOTE: we have to verify the offset at runtime. + VM_VerifyResultRegI32(value); + }); + + VERIFY_OP(CORE, GlobalStoreIndirectI32, { + VM_VerifyOperandRegI32(byte_offset); + // NOTE: we have to verify the offset at runtime. + VM_VerifyOperandRegI32(value); + }); + + VERIFY_OP(CORE, GlobalLoadI64, { + VM_VerifyGlobalAttr(byte_offset); + VM_VerifyRwdataOffset(byte_offset, 8); + VM_VerifyResultRegI64(value); + }); + + VERIFY_OP(CORE, GlobalStoreI64, { + VM_VerifyGlobalAttr(byte_offset); + VM_VerifyRwdataOffset(byte_offset, 8); + VM_VerifyOperandRegI64(value); + }); + + VERIFY_OP(CORE, GlobalLoadIndirectI64, { + VM_VerifyOperandRegI32(byte_offset); + // NOTE: we have to verify the offset at runtime. + VM_VerifyResultRegI64(value); + }); + + VERIFY_OP(CORE, GlobalStoreIndirectI64, { + VM_VerifyOperandRegI32(byte_offset); + // NOTE: we have to verify the offset at runtime. + VM_VerifyOperandRegI64(value); + }); + + VERIFY_OP(CORE, GlobalLoadRef, { + VM_VerifyGlobalAttr(global); + VM_VerifyGlobalRefOrdinal(global); + VM_VerifyTypeOf(type_def); + VM_VerifyResultRegRef(value); + }); + + VERIFY_OP(CORE, GlobalStoreRef, { + VM_VerifyGlobalAttr(global); + VM_VerifyGlobalRefOrdinal(global); + VM_VerifyTypeOf(type_def); + VM_VerifyOperandRegRef(value); + }); + + VERIFY_OP(CORE, GlobalLoadIndirectRef, { + VM_VerifyOperandRegI32(global); + // NOTE: we have to verify the ordinal at runtime. + VM_VerifyTypeOf(type_def); + VM_VerifyResultRegRef(value); + }); + + VERIFY_OP(CORE, GlobalStoreIndirectRef, { + VM_VerifyOperandRegI32(global); + // NOTE: we have to verify the ordinal at runtime. + VM_VerifyTypeOf(type_def); + VM_VerifyOperandRegRef(value); + }); + + //===------------------------------------------------------------------===// + // Constants + //===------------------------------------------------------------------===// + + VERIFY_OP(CORE, ConstI32, { + VM_VerifyIntAttr32(value); + VM_VerifyResultRegI32(result); + }); + + VERIFY_OP(CORE, ConstI32Zero, { VM_VerifyResultRegI32(result); }); + + VERIFY_OP(CORE, ConstI64, { + VM_VerifyIntAttr64(value); + VM_VerifyResultRegI64(result); + }); + + VERIFY_OP(CORE, ConstI64Zero, { VM_VerifyResultRegI64(result); }); + + VERIFY_OP(CORE, ConstRefZero, { VM_VerifyResultRegRef(result); }); + + VERIFY_OP(CORE, ConstRefRodata, { + VM_VerifyRodataAttr(rodata); + VM_VerifyRodataOrdinal(rodata); + VM_VerifyResultRegRef(value); + }); + + //===------------------------------------------------------------------===// + // Buffers + //===------------------------------------------------------------------===// + + VERIFY_OP(CORE, BufferAlloc, { + VM_VerifyOperandRegI64HostSize(length); + VM_VerifyResultRegRef(result); + }); + + VERIFY_OP(CORE, BufferClone, { + VM_VerifyOperandRegRef(source); + VM_VerifyOperandRegI64HostSize(offset); + VM_VerifyOperandRegI64HostSize(length); + VM_VerifyResultRegRef(result); + }); + + VERIFY_OP(CORE, BufferLength, { + VM_VerifyOperandRegRef(buffer); + VM_VerifyResultRegI64(result); + }); + + VERIFY_OP(CORE, BufferCopy, { + VM_VerifyOperandRegRef(source_buffer); + VM_VerifyOperandRegI64HostSize(source_offset); + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI64HostSize(length); + }); + + VERIFY_OP(CORE, BufferCompare, { + VM_VerifyOperandRegRef(lhs_buffer); + VM_VerifyOperandRegI64HostSize(lhs_offset); + VM_VerifyOperandRegRef(rhs_buffer); + VM_VerifyOperandRegI64HostSize(rhs_offset); + VM_VerifyOperandRegI64HostSize(length); + VM_VerifyResultRegI32(result); + }); + + VERIFY_OP(CORE, BufferFillI8, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI64HostSize(length); + VM_VerifyOperandRegI32(value); + }); + VERIFY_OP(CORE, BufferFillI16, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI64HostSize(length); + VM_VerifyOperandRegI32(value); + }); + VERIFY_OP(CORE, BufferFillI32, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI64HostSize(length); + VM_VerifyOperandRegI32(value); + }); + VERIFY_OP(CORE, BufferFillI64, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI64HostSize(length); + VM_VerifyOperandRegI64(value); + }); + + VERIFY_OP(CORE, BufferLoadI8U, { + VM_VerifyOperandRegRef(source_buffer); + VM_VerifyOperandRegI64HostSize(source_offset); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(CORE, BufferLoadI8S, { + VM_VerifyOperandRegRef(source_buffer); + VM_VerifyOperandRegI64HostSize(source_offset); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(CORE, BufferLoadI16U, { + VM_VerifyOperandRegRef(source_buffer); + VM_VerifyOperandRegI64HostSize(source_offset); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(CORE, BufferLoadI16S, { + VM_VerifyOperandRegRef(source_buffer); + VM_VerifyOperandRegI64HostSize(source_offset); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(CORE, BufferLoadI32, { + VM_VerifyOperandRegRef(source_buffer); + VM_VerifyOperandRegI64HostSize(source_offset); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(CORE, BufferLoadI64, { + VM_VerifyOperandRegRef(source_buffer); + VM_VerifyOperandRegI64HostSize(source_offset); + VM_VerifyResultRegI64(result); + }); + + VERIFY_OP(CORE, BufferStoreI8, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI32(value); + }); + VERIFY_OP(CORE, BufferStoreI16, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI32(value); + }); + VERIFY_OP(CORE, BufferStoreI32, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI32(value); + }); + VERIFY_OP(CORE, BufferStoreI64, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI64(value); + }); + + //===------------------------------------------------------------------===// + // Lists + //===------------------------------------------------------------------===// + + VERIFY_OP(CORE, ListAlloc, { + VM_VerifyTypeOf(element_type); + VM_VerifyOperandRegI32(initial_capacity); + VM_VerifyResultRegRef(result); + }); + + VERIFY_OP(CORE, ListReserve, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(minimum_capacity); + }); + + VERIFY_OP(CORE, ListSize, { + VM_VerifyOperandRegRef(list); + VM_VerifyResultRegI32(result); + }); + + VERIFY_OP(CORE, ListResize, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(new_size); + }); + + VERIFY_OP(CORE, ListGetI32, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(index); + VM_VerifyResultRegI32(result); + }); + + VERIFY_OP(CORE, ListSetI32, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(index); + VM_VerifyOperandRegI32(raw_value); + }); + + VERIFY_OP(CORE, ListGetI64, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(index); + VM_VerifyResultRegI64(result); + }); + + VERIFY_OP(CORE, ListSetI64, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(index); + VM_VerifyOperandRegI64(value); + }); + + VERIFY_OP(CORE, ListGetRef, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(index); + VM_VerifyTypeOf(type_def); + VM_VerifyResultRegRef(result); + }); + + VERIFY_OP(CORE, ListSetRef, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(index); + VM_VerifyOperandRegRef(value); + }); + + //===------------------------------------------------------------------===// + // Conditional assignment + //===------------------------------------------------------------------===// + + VERIFY_OP(CORE, SelectI32, { + VM_VerifyOperandRegI32(condition); + VM_VerifyOperandRegI32(true_value); + VM_VerifyOperandRegI32(false_value); + VM_VerifyResultRegI32(result); + }); + + VERIFY_OP(CORE, SelectI64, { + VM_VerifyOperandRegI32(condition); + VM_VerifyOperandRegI64(true_value); + VM_VerifyOperandRegI64(false_value); + VM_VerifyResultRegI64(result); + }); + + VERIFY_OP(CORE, SelectRef, { + VM_VerifyOperandRegI32(condition); + // TODO(benvanik): remove the type_id and use either LHS/RHS (if both are + // null then output is always null so no need to know the type). + VM_VerifyTypeOf(true_value_type_def); + VM_VerifyOperandRegRef(true_value); + VM_VerifyOperandRegRef(false_value); + VM_VerifyResultRegRef(result); + }); + + VERIFY_OP(CORE, SwitchI32, { + VM_VerifyOperandRegI32(index); + VM_VerifyIntAttr32(default_value); + VM_VerifyVariadicOperandsI32(values); + VM_VerifyResultRegI32(result); + }); + + VERIFY_OP(CORE, SwitchI64, { + VM_VerifyOperandRegI32(index); + VM_VerifyIntAttr64(default_value); + VM_VerifyVariadicOperandsI64(values); + VM_VerifyResultRegI64(result); + }); + + VERIFY_OP(CORE, SwitchRef, { + VM_VerifyOperandRegI32(index); + VM_VerifyTypeOf(type_def); + VM_VerifyOperandRegRef(default_value); + VM_VerifyVariadicOperandsRef(values, type_def); + VM_VerifyResultRegRef(result); + }); + + //===------------------------------------------------------------------===// + // Native integer arithmetic + //===------------------------------------------------------------------===// + + VERIFY_OP_CORE_BINARY_I32(AddI32); + VERIFY_OP_CORE_BINARY_I32(SubI32); + VERIFY_OP_CORE_BINARY_I32(MulI32); + VERIFY_OP_CORE_BINARY_I32(DivI32S); + VERIFY_OP_CORE_BINARY_I32(DivI32U); + VERIFY_OP_CORE_BINARY_I32(RemI32S); + VERIFY_OP_CORE_BINARY_I32(RemI32U); + VERIFY_OP_CORE_TERNARY_I32(FMAI32); + VERIFY_OP_CORE_UNARY_I32(AbsI32); + VERIFY_OP_CORE_UNARY_I32(NotI32); + VERIFY_OP_CORE_BINARY_I32(AndI32); + VERIFY_OP_CORE_BINARY_I32(OrI32); + VERIFY_OP_CORE_BINARY_I32(XorI32); + VERIFY_OP_CORE_UNARY_I32(CtlzI32); + + VERIFY_OP_CORE_BINARY_I64(AddI64); + VERIFY_OP_CORE_BINARY_I64(SubI64); + VERIFY_OP_CORE_BINARY_I64(MulI64); + VERIFY_OP_CORE_BINARY_I64(DivI64S); + VERIFY_OP_CORE_BINARY_I64(DivI64U); + VERIFY_OP_CORE_BINARY_I64(RemI64S); + VERIFY_OP_CORE_BINARY_I64(RemI64U); + VERIFY_OP_CORE_TERNARY_I64(FMAI64); + VERIFY_OP_CORE_UNARY_I64(AbsI64); + VERIFY_OP_CORE_UNARY_I64(NotI64); + VERIFY_OP_CORE_BINARY_I64(AndI64); + VERIFY_OP_CORE_BINARY_I64(OrI64); + VERIFY_OP_CORE_BINARY_I64(XorI64); + VERIFY_OP_CORE_UNARY_I64(CtlzI64); + + //===------------------------------------------------------------------===// + // Casting and type conversion/emulation + //===------------------------------------------------------------------===// + + // NOTE: these all operate on 32-bit registers. + VERIFY_OP_CORE_UNARY_I32(TruncI32I8); + VERIFY_OP_CORE_UNARY_I32(TruncI32I16); + VERIFY_OP_CORE_UNARY_I32(ExtI8I32S); + VERIFY_OP_CORE_UNARY_I32(ExtI8I32U); + VERIFY_OP_CORE_UNARY_I32(ExtI16I32S); + VERIFY_OP_CORE_UNARY_I32(ExtI16I32U); + + // NOTE: 64-bit ones are actually changing register widths. + VERIFY_OP(CORE, TruncI64I32, { + VM_VerifyOperandRegI64(operand); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(CORE, ExtI32I64S, { + VM_VerifyOperandRegI32(operand); + VM_VerifyResultRegI64(result); + }); + VERIFY_OP(CORE, ExtI32I64U, { + VM_VerifyOperandRegI32(operand); + VM_VerifyResultRegI64(result); + }); + + //===------------------------------------------------------------------===// + // Native bitwise shifts and rotates + //===------------------------------------------------------------------===// + +#define VERIFY_OP_CORE_SHIFT_I32(op_name) \ + VERIFY_OP(CORE, op_name, { \ + VM_VerifyOperandRegI32(operand); \ + VM_VerifyOperandRegI32(amount); \ + VM_VerifyResultRegI32(result); \ + }); + + VERIFY_OP_CORE_SHIFT_I32(ShlI32); + VERIFY_OP_CORE_SHIFT_I32(ShrI32S); + VERIFY_OP_CORE_SHIFT_I32(ShrI32U); + +#define VERIFY_OP_CORE_SHIFT_I64(op_name) \ + VERIFY_OP(CORE, op_name, { \ + VM_VerifyOperandRegI64(operand); \ + VM_VerifyOperandRegI32(amount); \ + VM_VerifyResultRegI64(result); \ + }); + + VERIFY_OP_CORE_SHIFT_I64(ShlI64); + VERIFY_OP_CORE_SHIFT_I64(ShrI64S); + VERIFY_OP_CORE_SHIFT_I64(ShrI64U); + + //===------------------------------------------------------------------===// + // Comparison ops + //===------------------------------------------------------------------===// + + VERIFY_OP_CORE_BINARY_I32(CmpEQI32); + VERIFY_OP_CORE_BINARY_I32(CmpNEI32); + VERIFY_OP_CORE_BINARY_I32(CmpLTI32S); + VERIFY_OP_CORE_BINARY_I32(CmpLTI32U); + VERIFY_OP_CORE_UNARY_I32(CmpNZI32); + +#define VERIFY_OP_CORE_CMP_I64(op_name) \ + VERIFY_OP(CORE, op_name, { \ + VM_VerifyOperandRegI64(lhs); \ + VM_VerifyOperandRegI64(rhs); \ + VM_VerifyResultRegI32(result); \ + }); + + VERIFY_OP_CORE_CMP_I64(CmpEQI64); + VERIFY_OP_CORE_CMP_I64(CmpNEI64); + VERIFY_OP_CORE_CMP_I64(CmpLTI64S); + VERIFY_OP_CORE_CMP_I64(CmpLTI64U); + VERIFY_OP(CORE, CmpNZI64, { + VM_VerifyOperandRegI64(operand); + VM_VerifyResultRegI32(result); + }); + + VERIFY_OP(CORE, CmpEQRef, { + VM_VerifyOperandRegRef(lhs); + VM_VerifyOperandRegRef(rhs); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(CORE, CmpNERef, { + VM_VerifyOperandRegRef(lhs); + VM_VerifyOperandRegRef(rhs); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(CORE, CmpNZRef, { + VM_VerifyOperandRegRef(operand); + VM_VerifyResultRegI32(result); + }); + + //===------------------------------------------------------------------===// + // Control flow + //===------------------------------------------------------------------===// + + VERIFY_OP(CORE, Block, { + // Define the new block in the block list. It may already be declared from + // a prior branch. + iree_vm_bytecode_block_t* block = NULL; + IREE_RETURN_IF_ERROR(iree_vm_bytecode_block_list_insert( + &verify_state->block_list, pc - 1, &block)); + block->defined = 1; + verify_state->in_block = 1; + }); + + VERIFY_OP(CORE, Branch, { + VM_VerifyBranchTarget(dest_pc); + VM_VerifyBranchOperands(operands); + verify_state->in_block = 0; // terminator + }); + + VERIFY_OP(CORE, CondBranch, { + VM_VerifyOperandRegI32(condition); + VM_VerifyBranchTarget(true_dest_pc); + VM_VerifyBranchOperands(true_operands); + VM_VerifyBranchTarget(false_dest_pc); + VM_VerifyBranchOperands(false_operands); + verify_state->in_block = 0; // terminator + }); + + VERIFY_OP(CORE, Call, { + VM_VerifyFuncAttr(callee_ordinal); + VM_VerifyVariadicOperandsAny(operands); + VM_VerifyVariadicResultsAny(results); + if (VM_IsImportOrdinal(callee_ordinal)) { + VM_UnmaskImportOrdinal(callee_ordinal); + VM_VerifyImportOrdinal(callee_ordinal); + iree_vm_ImportFunctionDef_table_t import_def = + iree_vm_ImportFunctionDef_vec_at(verify_state->imported_functions, + callee_ordinal); + IREE_RETURN_IF_ERROR( + iree_vm_bytecode_function_verify_call( + verify_state, iree_vm_ImportFunctionDef_signature(import_def), + /*segment_sizes=*/NULL, operands, results), + "call to import '%s'", + iree_vm_ImportFunctionDef_full_name(import_def)); + } else { + VM_VerifyFunctionOrdinal(callee_ordinal); + IREE_RETURN_IF_ERROR( + iree_vm_bytecode_function_verify_call( + verify_state, + iree_vm_FunctionSignatureDef_vec_at( + verify_state->function_signatures, callee_ordinal), + /*segment_sizes=*/NULL, operands, results), + "call to internal function %d", callee_ordinal); + } + }); + + VERIFY_OP(CORE, CallVariadic, { + VM_VerifyFuncAttr(callee_ordinal); + VM_VerifyVariadicOperands(segment_sizes); + VM_VerifyVariadicOperandsAny(operands); + VM_VerifyVariadicResultsAny(results); + if (IREE_UNLIKELY(!VM_IsImportOrdinal(callee_ordinal))) { + // Variadic calls are currently only supported for import functions. + return iree_make_status( + IREE_STATUS_FAILED_PRECONDITION, + "variadic calls only supported for internal callees"); + } + VM_UnmaskImportOrdinal(callee_ordinal); + VM_VerifyImportOrdinal(callee_ordinal); + iree_vm_ImportFunctionDef_table_t import_def = + iree_vm_ImportFunctionDef_vec_at(verify_state->imported_functions, + callee_ordinal); + IREE_RETURN_IF_ERROR( + iree_vm_bytecode_function_verify_call( + verify_state, iree_vm_ImportFunctionDef_signature(import_def), + segment_sizes, operands, results), + "variadic call to import '%s'", + iree_vm_ImportFunctionDef_full_name(import_def)); + }); + + VERIFY_OP(CORE, Return, { + VM_VerifyVariadicOperandsAny(operands); + IREE_RETURN_IF_ERROR(iree_vm_bytecode_function_verify_cconv_registers( + verify_state, verify_state->cconv_results, /*segment_sizes=*/NULL, + operands)); + verify_state->in_block = 0; // terminator + }); + + VERIFY_OP(CORE, Fail, { + VM_VerifyOperandRegI32(status); + iree_string_view_t message; + VM_VerifyStrAttr(message, &message); + verify_state->in_block = 0; // terminator + }); + + VERIFY_OP(CORE, ImportResolved, { + VM_VerifyFuncAttr(import_ordinal); + if (IREE_UNLIKELY(!VM_IsImportOrdinal(import_ordinal))) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "function ordinal %u is not an import ordinal", + import_ordinal); + } + VM_UnmaskImportOrdinal(import_ordinal); + VM_VerifyImportOrdinal(import_ordinal); + VM_VerifyResultRegI32(result); + }); + + //===------------------------------------------------------------------===// + // Async/fiber ops + //===------------------------------------------------------------------===// + + VERIFY_OP(CORE, Yield, { + VM_VerifyBranchTarget(dest_pc); + VM_VerifyBranchOperands(operands); + verify_state->in_block = 0; // terminator + }); + + //===------------------------------------------------------------------===// + // Debugging + //===------------------------------------------------------------------===// + + VERIFY_OP(CORE, Trace, { + iree_string_view_t event_name; + VM_VerifyStrAttr(event_name, &event_name); + VM_VerifyVariadicOperandsAny(operands); + }); + + VERIFY_OP(CORE, Print, { + iree_string_view_t event_name; + VM_VerifyStrAttr(event_name, &event_name); + VM_VerifyVariadicOperandsAny(operands); + }); + + VERIFY_OP(CORE, Break, { + VM_VerifyBranchTarget(dest_pc); + VM_VerifyBranchOperands(operands); + verify_state->in_block = 0; // terminator + }); + + VERIFY_OP(CORE, CondBreak, { + VM_VerifyOperandRegI32(condition); + VM_VerifyBranchTarget(dest); + VM_VerifyBranchOperands(operands); + verify_state->in_block = 0; // terminator + }); + + //===------------------------------------------------------------------===// + // Extension trampolines + //===------------------------------------------------------------------===// + +#if IREE_VM_EXT_F32_ENABLE + BEGIN_VERIFY_PREFIX(PrefixExtF32, iree_vm_FeatureBits_EXT_F32) + + //===----------------------------------------------------------------===// + // ExtF32: Globals + //===----------------------------------------------------------------===// + + VERIFY_OP(EXT_F32, GlobalLoadF32, { + VM_VerifyGlobalAttr(byte_offset); + VM_VerifyRwdataOffset(byte_offset, 4); + VM_VerifyResultRegF32(value); + }); + + VERIFY_OP(EXT_F32, GlobalStoreF32, { + VM_VerifyGlobalAttr(byte_offset); + VM_VerifyRwdataOffset(byte_offset, 4); + VM_VerifyOperandRegF32(value); + }); + + VERIFY_OP(EXT_F32, GlobalLoadIndirectF32, { + VM_VerifyOperandRegI32(byte_offset); + // NOTE: we have to verify the offset at runtime. + VM_VerifyResultRegF32(value); + }); + + VERIFY_OP(EXT_F32, GlobalStoreIndirectF32, { + VM_VerifyOperandRegI32(byte_offset); + // NOTE: we have to verify the offset at runtime. + VM_VerifyOperandRegF32(value); + }); + + //===----------------------------------------------------------------===// + // ExtF32: Constants + //===----------------------------------------------------------------===// + + VERIFY_OP(EXT_F32, ConstF32, { + VM_VerifyFloatAttr32(value); + VM_VerifyResultRegF32(result); + }); + + VERIFY_OP(EXT_F32, ConstF32Zero, { VM_VerifyResultRegF32(result); }); + + //===----------------------------------------------------------------===// + // ExtF32: Lists + //===----------------------------------------------------------------===// + + VERIFY_OP(EXT_F32, ListGetF32, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(index); + VM_VerifyResultRegF32(result); + }); + + VERIFY_OP(EXT_F32, ListSetF32, { + VM_VerifyOperandRegRef(list); + VM_VerifyOperandRegI32(index); + VM_VerifyOperandRegF32(value); + }); + + //===----------------------------------------------------------------===// + // ExtF32: Conditional assignment + //===----------------------------------------------------------------===// + + VERIFY_OP(EXT_F32, SelectF32, { + VM_VerifyOperandRegI32(condition); + VM_VerifyOperandRegF32(true_value); + VM_VerifyOperandRegF32(false_value); + VM_VerifyResultRegF32(result); + }); + + VERIFY_OP(EXT_F32, SwitchF32, { + VM_VerifyOperandRegI32(index); + VM_VerifyFloatAttr32(default_value); + VM_VerifyVariadicOperandsF32(values); + VM_VerifyResultRegF32(result); + }); + + //===----------------------------------------------------------------===// + // ExtF32: Native floating-point arithmetic + //===----------------------------------------------------------------===// + + VERIFY_OP_EXT_F32_BINARY_F32(AddF32); + VERIFY_OP_EXT_F32_BINARY_F32(SubF32); + VERIFY_OP_EXT_F32_BINARY_F32(MulF32); + VERIFY_OP_EXT_F32_BINARY_F32(DivF32); + VERIFY_OP_EXT_F32_BINARY_F32(RemF32); + VERIFY_OP_EXT_F32_TERNARY_F32(FMAF32); + VERIFY_OP_EXT_F32_UNARY_F32(AbsF32); + VERIFY_OP_EXT_F32_UNARY_F32(NegF32); + VERIFY_OP_EXT_F32_UNARY_F32(CeilF32); + VERIFY_OP_EXT_F32_UNARY_F32(FloorF32); + VERIFY_OP_EXT_F32_UNARY_F32(RoundF32); + + VERIFY_OP_EXT_F32_UNARY_F32(AtanF32); + VERIFY_OP_EXT_F32_BINARY_F32(Atan2F32); + VERIFY_OP_EXT_F32_UNARY_F32(CosF32); + VERIFY_OP_EXT_F32_UNARY_F32(SinF32); + VERIFY_OP_EXT_F32_UNARY_F32(ExpF32); + VERIFY_OP_EXT_F32_UNARY_F32(Exp2F32); + VERIFY_OP_EXT_F32_UNARY_F32(ExpM1F32); + VERIFY_OP_EXT_F32_UNARY_F32(LogF32); + VERIFY_OP_EXT_F32_UNARY_F32(Log10F32); + VERIFY_OP_EXT_F32_UNARY_F32(Log1pF32); + VERIFY_OP_EXT_F32_UNARY_F32(Log2F32); + VERIFY_OP_EXT_F32_BINARY_F32(PowF32); + VERIFY_OP_EXT_F32_UNARY_F32(RsqrtF32); + VERIFY_OP_EXT_F32_UNARY_F32(SqrtF32); + VERIFY_OP_EXT_F32_UNARY_F32(TanhF32); + VERIFY_OP_EXT_F32_UNARY_F32(ErfF32); + + //===----------------------------------------------------------------===// + // ExtF32: Casting and type conversion/emulation + //===----------------------------------------------------------------===// + + VERIFY_OP(EXT_F32, CastSI32F32, { + VM_VerifyOperandRegI32(operand); + VM_VerifyResultRegF32(result); + }); + VERIFY_OP(EXT_F32, CastUI32F32, { + VM_VerifyOperandRegI32(operand); + VM_VerifyResultRegF32(result); + }); + VERIFY_OP(EXT_F32, CastF32SI32, { + VM_VerifyOperandRegF32(operand); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(EXT_F32, CastF32UI32, { + VM_VerifyOperandRegF32(operand); + VM_VerifyResultRegI32(result); + }); + VERIFY_OP(EXT_F32, BitcastI32F32, { + VM_VerifyOperandRegI32(operand); + VM_VerifyResultRegF32(result); + }); + VERIFY_OP(EXT_F32, BitcastF32I32, { + VM_VerifyOperandRegF32(operand); + VM_VerifyResultRegI32(result); + }); + + //===----------------------------------------------------------------===// + // ExtF32: Comparison ops + //===----------------------------------------------------------------===// + +#define VERIFY_OP_EXT_F32_CMP_F32(op_name) \ + VERIFY_OP(EXT_F32, op_name, { \ + VM_VerifyOperandRegF32(lhs); \ + VM_VerifyOperandRegF32(rhs); \ + VM_VerifyResultRegI32(result); \ + }); + + VERIFY_OP_EXT_F32_CMP_F32(CmpEQF32O); + VERIFY_OP_EXT_F32_CMP_F32(CmpEQF32U); + VERIFY_OP_EXT_F32_CMP_F32(CmpNEF32O); + VERIFY_OP_EXT_F32_CMP_F32(CmpNEF32U); + VERIFY_OP_EXT_F32_CMP_F32(CmpLTF32O); + VERIFY_OP_EXT_F32_CMP_F32(CmpLTF32U); + VERIFY_OP_EXT_F32_CMP_F32(CmpLTEF32O); + VERIFY_OP_EXT_F32_CMP_F32(CmpLTEF32U); + VERIFY_OP(EXT_F32, CmpNaNF32, { + VM_VerifyOperandRegF32(operand); + VM_VerifyResultRegI32(result); + }); + + //===----------------------------------------------------------------===// + // ExtF32: Buffers + //===----------------------------------------------------------------===// + + VERIFY_OP(EXT_F32, BufferFillF32, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegI64HostSize(length); + VM_VerifyOperandRegF32(value); + }); + + VERIFY_OP(EXT_F32, BufferLoadF32, { + VM_VerifyOperandRegRef(source_buffer); + VM_VerifyOperandRegI64HostSize(source_offset); + VM_VerifyResultRegF32(result); + }); + + VERIFY_OP(EXT_F32, BufferStoreF32, { + VM_VerifyOperandRegRef(target_buffer); + VM_VerifyOperandRegI64HostSize(target_offset); + VM_VerifyOperandRegF32(value); + }); + + END_VERIFY_PREFIX(); +#else + UNHANDLED_VERIFY_PREFIX(PrefixExtF32, iree_vm_FeatureBits_EXT_F32); +#endif // IREE_VM_EXT_F32_ENABLE + + VERIFY_OP(CORE, PrefixExtF64, { + IREE_VM_VERIFY_REQUIREMENT(iree_vm_FeatureBits_EXT_F64); + return iree_make_status(IREE_STATUS_UNIMPLEMENTED, + "EXT_64 not yet implemented"); + }); + + default: + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "unrecognized opcode %u", bytecode_data[pc - 1]); + } + + *out_next_pc = pc; + return iree_ok_status(); +}
diff --git a/runtime/src/iree/vm/bytecode/verifier.h b/runtime/src/iree/vm/bytecode/verifier.h new file mode 100644 index 0000000..ae7f1af --- /dev/null +++ b/runtime/src/iree/vm/bytecode/verifier.h
@@ -0,0 +1,38 @@ +// Copyright 2023 The IREE Authors +// +// Licensed under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +#ifndef IREE_VM_BYTECODE_VERIFIER_H_ +#define IREE_VM_BYTECODE_VERIFIER_H_ + +#include "iree/base/api.h" +#include "iree/vm/api.h" +#include "iree/vm/bytecode/module_impl.h" + +// Verifies the structure of the FlatBuffer so that we can avoid doing so during +// runtime. There are still some conditions we must be aware of (such as omitted +// names on functions with internal linkage), however we shouldn't need to +// bounds check anything within the FlatBuffer after this succeeds. +iree_status_t iree_vm_bytecode_module_flatbuffer_verify( + iree_const_byte_span_t archive_contents, + iree_const_byte_span_t flatbuffer_contents, + iree_host_size_t archive_rodata_offset); + +// Verifies the bytecode contained within the given |function_ordinal|. +// Assumes that all information on |module| has been verified and only function +// information requires verification. +// +// NOTE: verification only checks that the function is well-formed and not that +// it is correct or will execute successfully. The only thing this tries to +// guarantee is that executing the bytecode won't cause a crash. +// +// If verification requires transient allocations for tracking they will be made +// from |scratch_allocator|. No allocation will live outside of the function and +// callers may provide stack-based arenas. +iree_status_t iree_vm_bytecode_function_verify( + iree_vm_bytecode_module_t* module, uint16_t function_ordinal, + iree_allocator_t scratch_allocator); + +#endif // IREE_VM_BYTECODE_VERIFIER_H_
diff --git a/runtime/src/iree/vm/list.c b/runtime/src/iree/vm/list.c index d7bdef5..0dafa55 100644 --- a/runtime/src/iree/vm/list.c +++ b/runtime/src/iree/vm/list.c
@@ -62,8 +62,6 @@ void* storage; }; -static iree_vm_ref_type_descriptor_t iree_vm_list_descriptor = {0}; - IREE_VM_DEFINE_TYPE_ADAPTERS(iree_vm_list, iree_vm_list_t); static void iree_vm_list_retain_range(iree_vm_list_t* list,
diff --git a/runtime/src/iree/vm/list_test.cc b/runtime/src/iree/vm/list_test.cc index e9b1d08..f20bc06 100644 --- a/runtime/src/iree/vm/list_test.cc +++ b/runtime/src/iree/vm/list_test.cc
@@ -23,7 +23,6 @@ private: float data_ = 1.0f; }; -static iree_vm_ref_type_descriptor_t test_a_descriptor = {0}; IREE_VM_DECLARE_TYPE_ADAPTERS(test_a, A); IREE_VM_DEFINE_TYPE_ADAPTERS(test_a, A); @@ -35,7 +34,6 @@ private: int data_ = 2; }; -static iree_vm_ref_type_descriptor_t test_b_descriptor = {0}; IREE_VM_DECLARE_TYPE_ADAPTERS(test_b, B); IREE_VM_DEFINE_TYPE_ADAPTERS(test_b, B);
diff --git a/runtime/src/iree/vm/module.c b/runtime/src/iree/vm/module.c index 30b0619..0be0bfa 100644 --- a/runtime/src/iree/vm/module.c +++ b/runtime/src/iree/vm/module.c
@@ -70,14 +70,14 @@ break; default: return iree_make_status(IREE_STATUS_UNIMPLEMENTED, - "unsupported cconv span type %c", + "unsupported cconv span type '%c'", cconv_fragment.data[i]); } } } break; default: return iree_make_status(IREE_STATUS_UNIMPLEMENTED, - "unsupported cconv type %c", + "unsupported cconv type '%c'", cconv_fragment.data[i]); } } @@ -159,7 +159,7 @@ break; default: return iree_make_status(IREE_STATUS_UNIMPLEMENTED, - "unsupported cconv span type %c", + "unsupported cconv span type '%c'", cconv_fragment.data[i]); } } @@ -167,7 +167,7 @@ } break; default: return iree_make_status(IREE_STATUS_UNIMPLEMENTED, - "unsupported cconv type %c", + "unsupported cconv type '%c'", cconv_fragment.data[i]); } } @@ -317,6 +317,7 @@ IREE_API_EXPORT iree_string_view_t iree_vm_function_name(const iree_vm_function_t* function) { IREE_ASSERT_ARGUMENT(function); + if (!function->module) return iree_string_view_empty(); iree_string_view_t name; iree_status_t status = function->module->get_function( function->module->self, function->linkage, function->ordinal,
diff --git a/runtime/src/iree/vm/ref.h b/runtime/src/iree/vm/ref.h index aa6e7fe..c2d1678 100644 --- a/runtime/src/iree/vm/ref.h +++ b/runtime/src/iree/vm/ref.h
@@ -239,23 +239,27 @@ // TODO(benvanik): make these macros standard/document them. #define IREE_VM_DECLARE_TYPE_ADAPTERS(name, T) \ + IREE_API_EXPORT_VARIABLE iree_vm_ref_type_descriptor_t name##_descriptor; \ + static inline iree_vm_ref_type_t name##_type_id() { \ + return name##_descriptor.type; \ + } \ + static inline bool name##_isa(const iree_vm_ref_t ref) { \ + return name##_descriptor.type == ref.type; \ + } \ IREE_API_EXPORT iree_vm_ref_t name##_retain_ref(T* value); \ IREE_API_EXPORT iree_vm_ref_t name##_move_ref(T* value); \ - IREE_API_EXPORT T* name##_deref(const iree_vm_ref_t ref); \ + static inline T* name##_deref(const iree_vm_ref_t ref) { \ + return IREE_LIKELY(name##_isa(ref)) ? (T*)ref.ptr : NULL; \ + } \ IREE_API_EXPORT iree_status_t name##_check_deref(const iree_vm_ref_t ref, \ T** out_ptr); \ IREE_API_EXPORT iree_status_t name##_check_deref_or_null( \ const iree_vm_ref_t ref, T** out_ptr); \ - IREE_API_EXPORT const iree_vm_ref_type_descriptor_t* \ - name##_get_descriptor(); \ - static inline bool name##_isa(const iree_vm_ref_t ref) { \ - return name##_get_descriptor()->type == ref.type; \ - } \ - IREE_API_EXPORT iree_vm_ref_type_t name##_type_id(); \ IREE_VM_DECLARE_CC_TYPE_LOOKUP(name, T) // TODO(benvanik): make these macros standard/document them. #define IREE_VM_DEFINE_TYPE_ADAPTERS(name, T) \ + iree_vm_ref_type_descriptor_t name##_descriptor = {0}; \ IREE_API_EXPORT iree_vm_ref_t name##_retain_ref(T* value) { \ iree_vm_ref_t ref = {0}; \ iree_vm_ref_wrap_retain(value, name##_descriptor.type, &ref); \ @@ -266,13 +270,6 @@ iree_vm_ref_wrap_assign(value, name##_descriptor.type, &ref); \ return ref; \ } \ - IREE_API_EXPORT T* name##_deref(const iree_vm_ref_t ref) { \ - if (IREE_UNLIKELY(ref.type != ref.type) || \ - IREE_UNLIKELY(ref.type == IREE_VM_REF_TYPE_NULL)) { \ - return NULL; \ - } \ - return (T*)ref.ptr; \ - } \ IREE_API_EXPORT iree_status_t name##_check_deref(const iree_vm_ref_t ref, \ T** out_ptr) { \ IREE_RETURN_IF_ERROR(iree_vm_ref_check(ref, name##_descriptor.type)); \ @@ -288,13 +285,6 @@ *out_ptr = NULL; \ } \ return iree_ok_status(); \ - } \ - IREE_API_EXPORT const iree_vm_ref_type_descriptor_t* \ - name##_get_descriptor() { \ - return &name##_descriptor; \ - } \ - IREE_API_EXPORT iree_vm_ref_type_t name##_type_id() { \ - return name##_descriptor.type; \ } // Optional C++ iree::vm::ref<T> wrapper.
diff --git a/runtime/src/iree/vm/ref_cc.h b/runtime/src/iree/vm/ref_cc.h index c43494a..f026fa9 100644 --- a/runtime/src/iree/vm/ref_cc.h +++ b/runtime/src/iree/vm/ref_cc.h
@@ -473,16 +473,16 @@ // dynamic type registration mechanism and that can be wrapped in an // iree_vm_ref_t. -#define IREE_VM_DECLARE_CC_TYPE_LOOKUP(name, T) \ - namespace iree { \ - namespace vm { \ - template <> \ - struct ref_type_descriptor<T> { \ - static const iree_vm_ref_type_descriptor_t* get() { \ - return name##_get_descriptor(); \ - } \ - }; \ - } \ +#define IREE_VM_DECLARE_CC_TYPE_LOOKUP(name, T) \ + namespace iree { \ + namespace vm { \ + template <> \ + struct ref_type_descriptor<T> { \ + static inline const iree_vm_ref_type_descriptor_t* get() { \ + return &name##_descriptor; \ + } \ + }; \ + } \ } #define IREE_VM_REGISTER_CC_TYPE(type, name, descriptor) \
diff --git a/runtime/src/iree/vm/stack.c b/runtime/src/iree/vm/stack.c index 6b39ed2..3d1b3a2 100644 --- a/runtime/src/iree/vm/stack.c +++ b/runtime/src/iree/vm/stack.c
@@ -16,12 +16,6 @@ #include "iree/base/tracing.h" #include "iree/vm/module.h" -#ifndef NDEBUG -#define VMCHECK(expr) assert(expr) -#else -#define VMCHECK(expr) -#endif // NDEBUG - //===----------------------------------------------------------------------===// // Stack implementation //===----------------------------------------------------------------------===// @@ -548,7 +542,7 @@ iree_vm_stack_t* stack, const iree_vm_function_t* function, iree_vm_stack_frame_type_t frame_type, iree_host_size_t frame_size, iree_vm_stack_frame_cleanup_fn_t frame_cleanup_fn, - iree_vm_stack_frame_t** out_callee_frame) { + iree_vm_stack_frame_t* IREE_RESTRICT* out_callee_frame) { if (out_callee_frame) *out_callee_frame = NULL; // Allocate stack space and grow stack, if required.
diff --git a/runtime/src/iree/vm/stack.h b/runtime/src/iree/vm/stack.h index f3aa557..7cb3253 100644 --- a/runtime/src/iree/vm/stack.h +++ b/runtime/src/iree/vm/stack.h
@@ -307,7 +307,7 @@ iree_vm_stack_t* stack, const iree_vm_function_t* function, iree_vm_stack_frame_type_t frame_type, iree_host_size_t frame_size, iree_vm_stack_frame_cleanup_fn_t frame_cleanup_fn, - iree_vm_stack_frame_t** out_callee_frame); + iree_vm_stack_frame_t* IREE_RESTRICT* out_callee_frame); // Leaves the current stack frame. IREE_API_EXPORT iree_status_t
diff --git a/samples/custom_module/basic/module.cc b/samples/custom_module/basic/module.cc index 6ba12d8..9f846d0 100644 --- a/samples/custom_module/basic/module.cc +++ b/samples/custom_module/basic/module.cc
@@ -22,11 +22,6 @@ // !custom.string type //===----------------------------------------------------------------------===// -// Runtime type descriptor for the !custom.string describing how to manage it -// and destroy it. The type ID is allocated at runtime and does not need to -// match the compiler ID. -static iree_vm_ref_type_descriptor_t iree_custom_string_descriptor = {0}; - // The "string" type we use to store and retain string data. // This could be arbitrarily complex or simply wrap another user-defined type. // The descriptor that is registered at startup defines how to manage the @@ -43,6 +38,9 @@ iree_string_view_t value; } iree_custom_string_t; +// Runtime type descriptor for the !custom.string describing how to manage it +// and destroy it. The type ID is allocated at runtime and does not need to +// match the compiler ID. IREE_VM_DEFINE_TYPE_ADAPTERS(iree_custom_string, iree_custom_string_t); extern "C" iree_status_t iree_custom_string_create(
diff --git a/samples/simple_embedding/simple_embedding.c b/samples/simple_embedding/simple_embedding.c index 3fa0c87..94272de 100644 --- a/samples/simple_embedding/simple_embedding.c +++ b/samples/simple_embedding/simple_embedding.c
@@ -127,7 +127,7 @@ // Get the result buffers from the invocation. iree_hal_buffer_view_t* ret_buffer_view = (iree_hal_buffer_view_t*)iree_vm_list_get_ref_deref( - outputs, 0, iree_hal_buffer_view_get_descriptor()); + outputs, 0, &iree_hal_buffer_view_descriptor); if (ret_buffer_view == NULL) { return iree_make_status(IREE_STATUS_NOT_FOUND, "can't find return buffer view");
diff --git a/tools/iree-dump-module-main.c b/tools/iree-dump-module-main.c index 9f4a9a0..c769270 100644 --- a/tools/iree-dump-module-main.c +++ b/tools/iree-dump-module-main.c
@@ -9,6 +9,7 @@ #include "iree/base/api.h" #include "iree/base/internal/file_io.h" #include "iree/schemas/bytecode_module_def_json_printer.h" +#include "iree/vm/bytecode/archive.h" #include "iree/vm/bytecode/module.h" // Today we just print to JSON. We could do something more useful (size @@ -28,7 +29,7 @@ &file_contents)); iree_const_byte_span_t flatbuffer_contents = iree_const_byte_span_empty(); - IREE_CHECK_OK(iree_vm_bytecode_module_parse_header( + IREE_CHECK_OK(iree_vm_bytecode_archive_parse_header( file_contents->const_buffer, &flatbuffer_contents, /*out_rodata_offset=*/NULL));