| // Copyright 2021 The IREE Authors | |
| // | |
| // Licensed under the Apache License v2.0 with LLVM Exceptions. | |
| // See https://llvm.org/LICENSE.txt for license information. | |
| // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception | |
| #include "experimental/rocm/rocm_buffer.h" | |
| #include <stddef.h> | |
| #include <stdint.h> | |
| #include <string.h> | |
| #include "experimental/rocm/rocm_allocator.h" | |
| #include "iree/base/api.h" | |
| #include "iree/base/tracing.h" | |
| typedef struct iree_hal_rocm_buffer_t { | |
| iree_hal_buffer_t base; | |
| void* host_ptr; | |
| hipDeviceptr_t device_ptr; | |
| } iree_hal_rocm_buffer_t; | |
| extern const iree_hal_buffer_vtable_t iree_hal_rocm_buffer_vtable; | |
| static iree_hal_rocm_buffer_t* iree_hal_rocm_buffer_cast( | |
| iree_hal_buffer_t* base_value) { | |
| IREE_HAL_ASSERT_TYPE(base_value, &iree_hal_rocm_buffer_vtable); | |
| return (iree_hal_rocm_buffer_t*)base_value; | |
| } | |
| iree_status_t iree_hal_rocm_buffer_wrap( | |
| iree_hal_allocator_t* allocator, iree_hal_memory_type_t memory_type, | |
| iree_hal_memory_access_t allowed_access, | |
| iree_hal_buffer_usage_t allowed_usage, iree_device_size_t allocation_size, | |
| iree_device_size_t byte_offset, iree_device_size_t byte_length, | |
| hipDeviceptr_t device_ptr, void* host_ptr, iree_hal_buffer_t** out_buffer) { | |
| IREE_ASSERT_ARGUMENT(allocator); | |
| IREE_ASSERT_ARGUMENT(out_buffer); | |
| IREE_TRACE_ZONE_BEGIN(z0); | |
| iree_hal_rocm_buffer_t* buffer = NULL; | |
| iree_status_t status = | |
| iree_allocator_malloc(iree_hal_allocator_host_allocator(allocator), | |
| sizeof(*buffer), (void**)&buffer); | |
| if (iree_status_is_ok(status)) { | |
| iree_hal_resource_initialize(&iree_hal_rocm_buffer_vtable, | |
| &buffer->base.resource); | |
| buffer->base.allocator = allocator; | |
| buffer->base.allocated_buffer = &buffer->base; | |
| buffer->base.allocation_size = allocation_size; | |
| buffer->base.byte_offset = byte_offset; | |
| buffer->base.byte_length = byte_length; | |
| buffer->base.memory_type = memory_type; | |
| buffer->base.allowed_access = allowed_access; | |
| buffer->base.allowed_usage = allowed_usage; | |
| buffer->host_ptr = host_ptr; | |
| buffer->device_ptr = device_ptr; | |
| *out_buffer = &buffer->base; | |
| } | |
| IREE_TRACE_ZONE_END(z0); | |
| return status; | |
| } | |
| static void iree_hal_rocm_buffer_destroy(iree_hal_buffer_t* base_buffer) { | |
| iree_hal_rocm_buffer_t* buffer = iree_hal_rocm_buffer_cast(base_buffer); | |
| iree_allocator_t host_allocator = | |
| iree_hal_allocator_host_allocator(iree_hal_buffer_allocator(base_buffer)); | |
| IREE_TRACE_ZONE_BEGIN(z0); | |
| iree_hal_rocm_allocator_free(buffer->base.allocator, buffer->base.memory_type, | |
| buffer->device_ptr, buffer->host_ptr, | |
| buffer->base.allocation_size); | |
| iree_allocator_free(host_allocator, buffer); | |
| IREE_TRACE_ZONE_END(z0); | |
| } | |
| static iree_status_t iree_hal_rocm_buffer_map_range( | |
| iree_hal_buffer_t* base_buffer, iree_hal_mapping_mode_t mapping_mode, | |
| iree_hal_memory_access_t memory_access, | |
| iree_device_size_t local_byte_offset, iree_device_size_t local_byte_length, | |
| void** out_data_ptr) { | |
| iree_hal_rocm_buffer_t* buffer = iree_hal_rocm_buffer_cast(base_buffer); | |
| if (!iree_all_bits_set(buffer->base.memory_type, | |
| IREE_HAL_MEMORY_TYPE_HOST_VISIBLE)) { | |
| return iree_make_status(IREE_STATUS_INTERNAL, | |
| "trying to map memory not host visible"); | |
| } | |
| uint8_t* data_ptr = (uint8_t*)(buffer->host_ptr) + local_byte_offset; | |
| // If we mapped for discard scribble over the bytes. This is not a mandated | |
| // behavior but it will make debugging issues easier. Alternatively for | |
| // heap buffers we could reallocate them such that ASAN yells, but that | |
| // would only work if the entire buffer was discarded. | |
| #ifndef NDEBUG | |
| if (iree_any_bit_set(memory_access, IREE_HAL_MEMORY_ACCESS_DISCARD)) { | |
| memset(data_ptr, 0xCD, local_byte_length); | |
| } | |
| #endif // !NDEBUG | |
| *out_data_ptr = data_ptr; | |
| return iree_ok_status(); | |
| } | |
| static void iree_hal_rocm_buffer_unmap_range( | |
| iree_hal_buffer_t* base_buffer, iree_device_size_t local_byte_offset, | |
| iree_device_size_t local_byte_length, void* data_ptr) { | |
| // nothing to do. | |
| } | |
| static iree_status_t iree_hal_rocm_buffer_invalidate_range( | |
| iree_hal_buffer_t* base_buffer, iree_device_size_t local_byte_offset, | |
| iree_device_size_t local_byte_length) { | |
| // Nothing to do. | |
| return iree_ok_status(); | |
| } | |
| static iree_status_t iree_hal_rocm_buffer_flush_range( | |
| iree_hal_buffer_t* base_buffer, iree_device_size_t local_byte_offset, | |
| iree_device_size_t local_byte_length) { | |
| // Nothing to do. | |
| return iree_ok_status(); | |
| } | |
| hipDeviceptr_t iree_hal_rocm_buffer_device_pointer( | |
| iree_hal_buffer_t* base_buffer) { | |
| iree_hal_rocm_buffer_t* buffer = iree_hal_rocm_buffer_cast(base_buffer); | |
| return buffer->device_ptr; | |
| } | |
| const iree_hal_buffer_vtable_t iree_hal_rocm_buffer_vtable = { | |
| .destroy = iree_hal_rocm_buffer_destroy, | |
| .map_range = iree_hal_rocm_buffer_map_range, | |
| .unmap_range = iree_hal_rocm_buffer_unmap_range, | |
| .invalidate_range = iree_hal_rocm_buffer_invalidate_range, | |
| .flush_range = iree_hal_rocm_buffer_flush_range, | |
| }; |