Optimizing vm.buffer access with inlined buffer data access. This allows performance sensitive places (emitc/interpreter) direct access to the buffer data in their optimization scope.
diff --git a/runtime/src/iree/vm/buffer.h b/runtime/src/iree/vm/buffer.h index 1d20984..250eb65 100644 --- a/runtime/src/iree/vm/buffer.h +++ b/runtime/src/iree/vm/buffer.h
@@ -188,6 +188,61 @@ iree_host_size_t target_offset, iree_host_size_t element_count, iree_host_size_t element_length); +// Low-level helper for accessing a typed view of a buffer for read access. +// The calling function must be safe to return from. Assumes buffer is non-null. +// Prefer iree_vm_buffer_read_elements for larger reads. +// +// Usage (read 4 floats from the buffer): +// const float* IREE_RESTRICT buffer_ptr = NULL; +// iree_vm_buffer_check_ro(buffer, offset, 4, float, buffer_ptr); +// process(buffer_ptr[0], buffer_ptr[1], buffer_ptr[2], buffer_ptr[3]); +#define iree_vm_buffer_check_ro(buffer, element_offset, element_length, \ + element_type, out_buffer_ptr) \ + { \ + const iree_host_size_t end = \ + ((element_offset) + (element_length)) * sizeof(element_type); \ + if (IREE_UNLIKELY(end > buffer->data.data_length)) { \ + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ + "out-of-bounds access detected (offset=%zu, " \ + "length=%zu, alignment=%zu, buffer length=%zu)", \ + (element_offset) * sizeof(element_type), \ + (element_length) * sizeof(element_type), \ + sizeof(element_type), buffer->data.data_length); \ + } \ + out_buffer_ptr = \ + (const element_type*)buffer->data.data + (element_offset); \ + } + +// Low-level helper for accessing a typed view of a buffer for write access. +// The calling function must be safe to return from. Assumes buffer is non-null. +// Prefer iree_vm_buffer_write_elements for larger reads. +// +// Usage (write a single float to the buffer): +// float* IREE_RESTRICT buffer_ptr = NULL; +// iree_vm_buffer_check_rw(buffer, offset, 1, float, buffer_ptr); +// buffer_ptr[0] = 1.0f; +#define iree_vm_buffer_check_rw(buffer, element_offset, element_length, \ + element_type, out_buffer_ptr) \ + { \ + if (IREE_UNLIKELY(!iree_all_bits_set(buffer->access, \ + IREE_VM_BUFFER_ACCESS_MUTABLE))) { \ + return iree_make_status( \ + IREE_STATUS_PERMISSION_DENIED, \ + "buffer is read-only and cannot be mapped for mutation"); \ + } \ + const iree_host_size_t end = \ + ((element_offset) + (element_length)) * sizeof(element_type); \ + if (IREE_UNLIKELY(end > buffer->data.data_length)) { \ + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, \ + "out-of-bounds access detected (offset=%zu, " \ + "length=%zu, alignment=%zu, buffer length=%zu)", \ + (element_offset) * sizeof(element_type), \ + (element_length) * sizeof(element_type), \ + sizeof(element_type), buffer->data.data_length); \ + } \ + out_buffer_ptr = (element_type*)buffer->data.data + (element_offset); \ + } + // Returns the a string view referencing the given |value| buffer. // The returned view will only be valid for as long as the buffer is live. static inline iree_string_view_t iree_vm_buffer_as_string(
diff --git a/runtime/src/iree/vm/bytecode/dispatch.c b/runtime/src/iree/vm/bytecode/dispatch.c index 5e2dbea..94c987d 100644 --- a/runtime/src/iree/vm/bytecode/dispatch.c +++ b/runtime/src/iree/vm/bytecode/dispatch.c
@@ -1062,7 +1062,7 @@ iree_host_size_t offset = VM_DecOperandRegI64HostSize("target_offset"); iree_host_size_t length = VM_DecOperandRegI64HostSize("length"); uint8_t value = (uint8_t)VM_DecOperandRegI32("value"); - IREE_RETURN_IF_ERROR(vm_buffer_fill_i8(buffer, offset, length, value)); + vm_buffer_fill_i8_inline(buffer, offset, length, value); }); DISPATCH_OP(CORE, BufferFillI16, { bool buffer_is_move; @@ -1075,7 +1075,7 @@ iree_host_size_t offset = VM_DecOperandRegI64HostSize("target_offset"); iree_host_size_t length = VM_DecOperandRegI64HostSize("length"); uint16_t value = (uint16_t)VM_DecOperandRegI32("value"); - IREE_RETURN_IF_ERROR(vm_buffer_fill_i16(buffer, offset, length, value)); + vm_buffer_fill_i16_inline(buffer, offset, length, value); }); DISPATCH_OP(CORE, BufferFillI32, { bool buffer_is_move; @@ -1088,7 +1088,7 @@ iree_host_size_t offset = VM_DecOperandRegI64HostSize("target_offset"); iree_host_size_t length = VM_DecOperandRegI64HostSize("length"); uint32_t value = VM_DecOperandRegI32("value"); - IREE_RETURN_IF_ERROR(vm_buffer_fill_i32(buffer, offset, length, value)); + vm_buffer_fill_i32_inline(buffer, offset, length, value); }); DISPATCH_OP(CORE, BufferFillI64, { bool buffer_is_move; @@ -1101,7 +1101,7 @@ iree_host_size_t offset = VM_DecOperandRegI64HostSize("target_offset"); iree_host_size_t length = VM_DecOperandRegI64HostSize("length"); uint64_t value = VM_DecOperandRegI64("value"); - IREE_RETURN_IF_ERROR(vm_buffer_fill_i64(buffer, offset, length, value)); + vm_buffer_fill_i64_inline(buffer, offset, length, value); }); // TODO(benvanik): rework dispatch so that the LoadI* ops can share the same @@ -1119,7 +1119,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("source_offset"); uint32_t* result = VM_DecResultRegI32("result"); - IREE_RETURN_IF_ERROR(vm_buffer_load_i8u(buffer, offset, result)); + vm_buffer_load_i8u_inline(buffer, offset, result); }); DISPATCH_OP(CORE, BufferLoadI8S, { bool buffer_is_move; @@ -1132,7 +1132,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("source_offset"); uint32_t* result = VM_DecResultRegI32("result"); - IREE_RETURN_IF_ERROR(vm_buffer_load_i8s(buffer, offset, result)); + vm_buffer_load_i8s_inline(buffer, offset, result); }); DISPATCH_OP(CORE, BufferLoadI16U, { bool buffer_is_move; @@ -1145,7 +1145,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("source_offset"); uint32_t* result = VM_DecResultRegI32("result"); - IREE_RETURN_IF_ERROR(vm_buffer_load_i16u(buffer, offset, result)); + vm_buffer_load_i16u_inline(buffer, offset, result); }); DISPATCH_OP(CORE, BufferLoadI16S, { bool buffer_is_move; @@ -1158,7 +1158,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("source_offset"); uint32_t* result = VM_DecResultRegI32("result"); - IREE_RETURN_IF_ERROR(vm_buffer_load_i16s(buffer, offset, result)); + vm_buffer_load_i16s_inline(buffer, offset, result); }); DISPATCH_OP(CORE, BufferLoadI32, { bool buffer_is_move; @@ -1171,7 +1171,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("source_offset"); uint32_t* result = VM_DecResultRegI32("result"); - IREE_RETURN_IF_ERROR(vm_buffer_load_i32(buffer, offset, result)); + vm_buffer_load_i32_inline(buffer, offset, result); }); DISPATCH_OP(CORE, BufferLoadI64, { bool buffer_is_move; @@ -1184,7 +1184,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("source_offset"); uint64_t* result = VM_DecResultRegI64("result"); - IREE_RETURN_IF_ERROR(vm_buffer_load_i64(buffer, offset, result)); + vm_buffer_load_i64_inline(buffer, offset, result); }); // TODO(benvanik): rework dispatch so that the StoreI* ops can share the @@ -1201,7 +1201,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("target_offset"); uint8_t value = (uint8_t)VM_DecOperandRegI32("value"); - IREE_RETURN_IF_ERROR(vm_buffer_store_i8(buffer, offset, value)); + vm_buffer_store_i8_inline(buffer, offset, value); }); DISPATCH_OP(CORE, BufferStoreI16, { bool buffer_is_move; @@ -1214,7 +1214,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("target_offset"); uint16_t value = (uint16_t)VM_DecOperandRegI32("value"); - IREE_RETURN_IF_ERROR(vm_buffer_store_i16(buffer, offset, value)); + vm_buffer_store_i16_inline(buffer, offset, value); }); DISPATCH_OP(CORE, BufferStoreI32, { bool buffer_is_move; @@ -1227,7 +1227,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("target_offset"); uint32_t value = VM_DecOperandRegI32("value"); - IREE_RETURN_IF_ERROR(vm_buffer_store_i32(buffer, offset, value)); + vm_buffer_store_i32_inline(buffer, offset, value); }); DISPATCH_OP(CORE, BufferStoreI64, { bool buffer_is_move; @@ -1240,7 +1240,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("target_offset"); uint64_t value = (uint64_t)VM_DecOperandRegI64("value"); - IREE_RETURN_IF_ERROR(vm_buffer_store_i64(buffer, offset, value)); + vm_buffer_store_i64_inline(buffer, offset, value); }); //===------------------------------------------------------------------===// @@ -2098,7 +2098,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("source_offset"); float* result = VM_DecResultRegF32("result"); - IREE_RETURN_IF_ERROR(vm_buffer_load_f32(buffer, offset, result)); + vm_buffer_load_f32_inline(buffer, offset, result); }); DISPATCH_OP(EXT_F32, BufferStoreF32, { @@ -2112,7 +2112,7 @@ } iree_host_size_t offset = VM_DecOperandRegI64HostSize("target_offset"); float value = VM_DecOperandRegF32("value"); - IREE_RETURN_IF_ERROR(vm_buffer_store_f32(buffer, offset, value)); + vm_buffer_store_f32_inline(buffer, offset, value); }); } END_DISPATCH_PREFIX();
diff --git a/runtime/src/iree/vm/ops.h b/runtime/src/iree/vm/ops.h index 9351420..1bfffbd 100644 --- a/runtime/src/iree/vm/ops.h +++ b/runtime/src/iree/vm/ops.h
@@ -59,118 +59,169 @@ return iree_ok_status(); } +#define vm_buffer_fill_inline(buffer, element_offset, element_length, \ + element_type, value) \ + element_type* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_rw(buffer, element_offset, element_length, \ + element_type, buffer_ptr); \ + for (iree_host_size_t i = 0; i < element_length; ++i) { \ + buffer_ptr[i] = value; \ + } + +#define vm_buffer_fill_i8_inline(buffer, element_offset, element_length, \ + value) \ + uint8_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_rw(buffer, offset, length, uint8_t, buffer_ptr); \ + memset(buffer_ptr, value, length); static inline iree_status_t vm_buffer_fill_i8(iree_vm_buffer_t* buffer, iree_host_size_t offset, iree_host_size_t length, uint8_t value) { - return iree_vm_buffer_fill_elements(buffer, offset, length, sizeof(value), - &value); + vm_buffer_fill_i8_inline(buffer, offset, length, value); + return iree_ok_status(); } +#define vm_buffer_fill_i16_inline(buffer, element_offset, element_length, \ + value) \ + vm_buffer_fill_inline(buffer, element_offset, element_length, uint16_t, value) static inline iree_status_t vm_buffer_fill_i16(iree_vm_buffer_t* buffer, iree_host_size_t offset, iree_host_size_t length, uint16_t value) { - return iree_vm_buffer_fill_elements(buffer, offset, length, sizeof(value), - &value); + vm_buffer_fill_i16_inline(buffer, offset, length, value); + return iree_ok_status(); } +#define vm_buffer_fill_i32_inline(buffer, element_offset, element_length, \ + value) \ + vm_buffer_fill_inline(buffer, element_offset, element_length, uint32_t, value) static inline iree_status_t vm_buffer_fill_i32(iree_vm_buffer_t* buffer, iree_host_size_t offset, iree_host_size_t length, uint32_t value) { - return iree_vm_buffer_fill_elements(buffer, offset, length, sizeof(value), - &value); + vm_buffer_fill_i32_inline(buffer, offset, length, value); + return iree_ok_status(); } +#define vm_buffer_fill_i64_inline(buffer, element_offset, element_length, \ + value) \ + vm_buffer_fill_inline(buffer, element_offset, element_length, uint64_t, value) static inline iree_status_t vm_buffer_fill_i64(iree_vm_buffer_t* buffer, iree_host_size_t offset, iree_host_size_t length, uint64_t value) { - return iree_vm_buffer_fill_elements(buffer, offset, length, sizeof(value), - &value); + vm_buffer_fill_i64_inline(buffer, offset, length, value); + return iree_ok_status(); } +#define vm_buffer_load_i8u_inline(buffer, element_offset, result) \ + const uint8_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_ro(buffer, element_offset, 1, uint8_t, buffer_ptr); \ + *result = vm_ext_i8i32u(*buffer_ptr); static inline iree_status_t vm_buffer_load_i8u(iree_vm_buffer_t* buffer, iree_host_size_t offset, int32_t* result) { - uint8_t result_x8 = 0; - IREE_RETURN_IF_ERROR(iree_vm_buffer_read_elements(buffer, offset, &result_x8, - 1, sizeof(result_x8))); - *result = vm_ext_i8i32u(result_x8); + vm_buffer_load_i8u_inline(buffer, offset, result); return iree_ok_status(); } +#define vm_buffer_load_i8s_inline(buffer, element_offset, result) \ + const int8_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_ro(buffer, element_offset, 1, int8_t, buffer_ptr); \ + *result = vm_ext_i8i32s(*buffer_ptr); static inline iree_status_t vm_buffer_load_i8s(iree_vm_buffer_t* buffer, iree_host_size_t offset, int32_t* result) { - int8_t result_x8 = 0; - IREE_RETURN_IF_ERROR(iree_vm_buffer_read_elements(buffer, offset, &result_x8, - 1, sizeof(result_x8))); - *result = vm_ext_i8i32s(result_x8); + vm_buffer_load_i8s_inline(buffer, offset, result); return iree_ok_status(); } +#define vm_buffer_load_i16u_inline(buffer, element_offset, result) \ + const int16_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_ro(buffer, element_offset, 1, int16_t, buffer_ptr); \ + *result = vm_ext_i16i32u(*buffer_ptr); static inline iree_status_t vm_buffer_load_i16u(iree_vm_buffer_t* buffer, iree_host_size_t offset, int32_t* result) { - uint16_t result_x16 = 0; - IREE_RETURN_IF_ERROR(iree_vm_buffer_read_elements(buffer, offset, &result_x16, - 1, sizeof(result_x16))); - *result = vm_ext_i16i32u(result_x16); + vm_buffer_load_i16u_inline(buffer, offset, result); return iree_ok_status(); } +#define vm_buffer_load_i16s_inline(buffer, element_offset, result) \ + const int16_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_ro(buffer, element_offset, 1, int16_t, buffer_ptr); \ + *result = vm_ext_i16i32s(*buffer_ptr); static inline iree_status_t vm_buffer_load_i16s(iree_vm_buffer_t* buffer, iree_host_size_t offset, int32_t* result) { - int16_t result_x16 = 0; - IREE_RETURN_IF_ERROR(iree_vm_buffer_read_elements(buffer, offset, &result_x16, - 1, sizeof(result_x16))); - *result = vm_ext_i16i32s(result_x16); + vm_buffer_load_i16s_inline(buffer, offset, result); return iree_ok_status(); } +#define vm_buffer_load_i32_inline(buffer, element_offset, result) \ + const int32_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_ro(buffer, element_offset, 1, int32_t, buffer_ptr); \ + *result = *buffer_ptr; static inline iree_status_t vm_buffer_load_i32(iree_vm_buffer_t* buffer, iree_host_size_t offset, int32_t* result) { - return iree_vm_buffer_read_elements(buffer, offset, result, 1, - sizeof(*result)); + vm_buffer_load_i32_inline(buffer, offset, result); + return iree_ok_status(); } +#define vm_buffer_load_i64_inline(buffer, element_offset, result) \ + const int64_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_ro(buffer, element_offset, 1, int64_t, buffer_ptr); \ + *result = *buffer_ptr; static inline iree_status_t vm_buffer_load_i64(iree_vm_buffer_t* buffer, iree_host_size_t offset, int64_t* result) { - return iree_vm_buffer_read_elements(buffer, offset, result, 1, - sizeof(*result)); + vm_buffer_load_i64_inline(buffer, offset, result); + return iree_ok_status(); } +#define vm_buffer_store_i8_inline(buffer, element_offset, value) \ + uint8_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_rw(buffer, element_offset, 1, uint8_t, buffer_ptr); \ + *buffer_ptr = value; static inline iree_status_t vm_buffer_store_i8(iree_vm_buffer_t* buffer, iree_host_size_t offset, uint8_t value) { - return iree_vm_buffer_write_elements(&value, buffer, offset, 1, - sizeof(value)); + vm_buffer_store_i8_inline(buffer, offset, value); + return iree_ok_status(); } +#define vm_buffer_store_i16_inline(buffer, element_offset, value) \ + uint16_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_rw(buffer, element_offset, 1, uint16_t, buffer_ptr); \ + *buffer_ptr = value; static inline iree_status_t vm_buffer_store_i16(iree_vm_buffer_t* buffer, iree_host_size_t offset, uint16_t value) { - return iree_vm_buffer_write_elements(&value, buffer, offset, 1, - sizeof(value)); + vm_buffer_store_i16_inline(buffer, offset, value); + return iree_ok_status(); } +#define vm_buffer_store_i32_inline(buffer, element_offset, value) \ + uint32_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_rw(buffer, element_offset, 1, uint32_t, buffer_ptr); \ + *buffer_ptr = value; static inline iree_status_t vm_buffer_store_i32(iree_vm_buffer_t* buffer, iree_host_size_t offset, uint32_t value) { - return iree_vm_buffer_write_elements(&value, buffer, offset, 1, - sizeof(value)); + vm_buffer_store_i32_inline(buffer, offset, value); + return iree_ok_status(); } +#define vm_buffer_store_i64_inline(buffer, element_offset, value) \ + uint64_t* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_rw(buffer, element_offset, 1, uint64_t, buffer_ptr); \ + *buffer_ptr = value; static inline iree_status_t vm_buffer_store_i64(iree_vm_buffer_t* buffer, iree_host_size_t offset, uint64_t value) { - return iree_vm_buffer_write_elements(&value, buffer, offset, 1, - sizeof(value)); + vm_buffer_store_i64_inline(buffer, offset, value); + return iree_ok_status(); } //===------------------------------------------------------------------===// @@ -411,26 +462,37 @@ // ExtF32: Buffers //===------------------------------------------------------------------===// +#define vm_buffer_fill_f32_inline(buffer, element_offset, element_length, \ + value) \ + vm_buffer_fill_inline(buffer, element_offset, element_length, float, value) static inline iree_status_t vm_buffer_fill_f32(iree_vm_buffer_t* buffer, iree_host_size_t offset, iree_host_size_t length, float value) { - return iree_vm_buffer_fill_elements(buffer, offset, length, sizeof(value), - &value); + vm_buffer_fill_f32_inline(buffer, offset, length, value); + return iree_ok_status(); } +#define vm_buffer_load_f32_inline(buffer, element_offset, result) \ + const float* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_ro(buffer, element_offset, 1, float, buffer_ptr); \ + *result = *buffer_ptr; static inline iree_status_t vm_buffer_load_f32(iree_vm_buffer_t* buffer, iree_host_size_t offset, float* result) { - return iree_vm_buffer_read_elements(buffer, offset, result, 1, - sizeof(*result)); + vm_buffer_load_f32_inline(buffer, offset, result); + return iree_ok_status(); } +#define vm_buffer_store_f32_inline(buffer, element_offset, value) \ + float* IREE_RESTRICT buffer_ptr = NULL; \ + iree_vm_buffer_check_rw(buffer, element_offset, 1, float, buffer_ptr); \ + *buffer_ptr = value; static inline iree_status_t vm_buffer_store_f32(iree_vm_buffer_t* buffer, iree_host_size_t offset, float value) { - return iree_vm_buffer_write_elements(&value, buffer, offset, 1, - sizeof(value)); + vm_buffer_store_f32_inline(buffer, offset, value); + return iree_ok_status(); } //===------------------------------------------------------------------===//