Adding iree_hal_encoding_t enum and plumbing through runtime code. All locations have been updated to pass in DENSE_ROW_MAJOR as it is the current assumption. Progress on #6762.
diff --git a/bindings/python/iree/runtime/hal.h b/bindings/python/iree/runtime/hal.h index d25c5a6..25a1ff3 100644 --- a/bindings/python/iree/runtime/hal.h +++ b/bindings/python/iree/runtime/hal.h
@@ -100,9 +100,11 @@ iree_hal_buffer_view_t* bv; iree_hal_element_type_t element_type = iree_hal_make_element_type( IREE_HAL_ELEMENT_TYPE_NONE, element_size * 8); + iree_hal_encoding_type_t encoding_type = + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR; CheckApiStatus( iree_hal_buffer_view_create(raw_ptr(), shape.s.data(), shape.s.size(), - element_type, &bv), + element_type, encoding_type, &bv), "Error creating buffer view"); return HalBufferView::CreateRetained(bv); }
diff --git a/bindings/python/iree/runtime/vm.cc b/bindings/python/iree/runtime/vm.cc index 864bf5d..a9152c5 100644 --- a/bindings/python/iree/runtime/vm.cc +++ b/bindings/python/iree/runtime/vm.cc
@@ -238,13 +238,16 @@ "Dependent buffer arguments not implemented"); } + iree_hal_encoding_type_t encoding_type = + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR; + // Create the buffer_view. (note that numpy shape is ssize_t) std::vector<int> dims(py_view.ndim); std::copy(py_view.shape, py_view.shape + py_view.ndim, dims.begin()); iree_hal_buffer_view_t* buffer_view; CheckApiStatus( iree_hal_buffer_view_create(raw_buffer, dims.data(), dims.size(), - element_type, &buffer_view), + element_type, encoding_type, &buffer_view), "Error allocating buffer_view"); iree_hal_buffer_release(raw_buffer); iree_vm_ref_t buffer_view_ref = iree_hal_buffer_view_move_ref(buffer_view);
diff --git a/bindings/tflite/tensor.c b/bindings/tflite/tensor.c index 8633a2d..1fdb16d 100644 --- a/bindings/tflite/tensor.c +++ b/bindings/tflite/tensor.c
@@ -122,8 +122,9 @@ } iree_device_size_t allocation_size = 0; IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_hal_buffer_compute_view_size(shape_dims, tensor->shape_rank, - element_type, &allocation_size)); + z0, iree_hal_buffer_compute_view_size( + shape_dims, tensor->shape_rank, element_type, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, &allocation_size)); allocation_size *= storage_scalar; // If the old buffer is the same size then no need to realloc.
diff --git a/iree/hal/buffer_view.c b/iree/hal/buffer_view.c index af464d2..7539dec 100644 --- a/iree/hal/buffer_view.c +++ b/iree/hal/buffer_view.c
@@ -19,6 +19,7 @@ iree_atomic_ref_count_t ref_count; iree_hal_buffer_t* buffer; iree_hal_element_type_t element_type; + iree_hal_encoding_type_t encoding_type; iree_device_size_t byte_length; iree_host_size_t shape_rank; iree_hal_dim_t shape[]; @@ -27,6 +28,7 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_create( iree_hal_buffer_t* buffer, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, + iree_hal_encoding_type_t encoding_type, iree_hal_buffer_view_t** out_buffer_view) { IREE_ASSERT_ARGUMENT(buffer); IREE_ASSERT_ARGUMENT(out_buffer_view); @@ -54,6 +56,7 @@ buffer_view->buffer = buffer; iree_hal_buffer_retain(buffer_view->buffer); buffer_view->element_type = element_type; + buffer_view->encoding_type = encoding_type; buffer_view->byte_length = iree_hal_element_byte_count(buffer_view->element_type); buffer_view->shape_rank = shape_rank; @@ -94,7 +97,8 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_allocate_buffer( iree_hal_allocator_t* allocator, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, - iree_hal_memory_type_t memory_type, iree_hal_buffer_usage_t allowed_usage, + iree_hal_encoding_type_t encoding_type, iree_hal_memory_type_t memory_type, + iree_hal_buffer_usage_t allowed_usage, iree_hal_buffer_view_t** out_buffer_view) { IREE_ASSERT_ARGUMENT(allocator); IREE_ASSERT_ARGUMENT(out_buffer_view); @@ -102,7 +106,7 @@ iree_device_size_t allocation_size = 0; iree_status_t status = iree_hal_buffer_compute_view_size( - shape, shape_rank, element_type, &allocation_size); + shape, shape_rank, element_type, encoding_type, &allocation_size); iree_hal_buffer_t* buffer = NULL; if (iree_status_is_ok(status)) { @@ -111,8 +115,9 @@ } if (iree_status_is_ok(status)) { - status = iree_hal_buffer_view_create(buffer, shape, shape_rank, - element_type, out_buffer_view); + status = + iree_hal_buffer_view_create(buffer, shape, shape_rank, element_type, + encoding_type, out_buffer_view); } iree_hal_buffer_release(buffer); @@ -123,8 +128,9 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_clone_heap_buffer( iree_hal_allocator_t* allocator, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, - iree_hal_memory_type_t memory_type, iree_hal_buffer_usage_t allowed_usage, - iree_const_byte_span_t data, iree_hal_buffer_view_t** out_buffer_view) { + iree_hal_encoding_type_t encoding_type, iree_hal_memory_type_t memory_type, + iree_hal_buffer_usage_t allowed_usage, iree_const_byte_span_t data, + iree_hal_buffer_view_t** out_buffer_view) { IREE_ASSERT_ARGUMENT(allocator); IREE_ASSERT_ARGUMENT(out_buffer_view); IREE_TRACE_ZONE_BEGIN(z0); @@ -132,9 +138,9 @@ // Allocate the buffer. iree_hal_buffer_view_t* buffer_view = NULL; IREE_RETURN_AND_END_ZONE_IF_ERROR( - z0, iree_hal_buffer_view_allocate_buffer(allocator, shape, shape_rank, - element_type, memory_type, - allowed_usage, &buffer_view)); + z0, iree_hal_buffer_view_allocate_buffer( + allocator, shape, shape_rank, element_type, encoding_type, + memory_type, allowed_usage, &buffer_view)); // Copy all of the data into it in the worst way possible. // If you find yourself coming here from profiling: @@ -156,7 +162,8 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_wrap_heap_buffer( iree_hal_allocator_t* allocator, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, - iree_hal_memory_type_t memory_type, iree_hal_memory_access_t allowed_access, + iree_hal_encoding_type_t encoding_type, iree_hal_memory_type_t memory_type, + iree_hal_memory_access_t allowed_access, iree_hal_buffer_usage_t allowed_usage, iree_byte_span_t data, iree_allocator_t data_allocator, iree_hal_buffer_view_t** out_buffer_view) { IREE_ASSERT_ARGUMENT(allocator); @@ -170,8 +177,9 @@ data_allocator, &buffer); if (iree_status_is_ok(status)) { - status = iree_hal_buffer_view_create(buffer, shape, shape_rank, - element_type, out_buffer_view); + status = + iree_hal_buffer_view_create(buffer, shape, shape_rank, element_type, + encoding_type, out_buffer_view); } iree_hal_buffer_release(buffer); @@ -182,7 +190,8 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_wrap_or_clone_heap_buffer( iree_hal_allocator_t* allocator, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, - iree_hal_memory_type_t memory_type, iree_hal_memory_access_t allowed_access, + iree_hal_encoding_type_t encoding_type, iree_hal_memory_type_t memory_type, + iree_hal_memory_access_t allowed_access, iree_hal_buffer_usage_t allowed_usage, iree_byte_span_t data, iree_allocator_t data_allocator, iree_hal_buffer_view_t** out_buffer_view) { IREE_ASSERT_ARGUMENT(allocator); @@ -200,12 +209,12 @@ compatibility, IREE_HAL_BUFFER_COMPATIBILITY_IMPORTABLE); if (wrap_allowed) { return iree_hal_buffer_view_wrap_heap_buffer( - allocator, shape, shape_rank, element_type, memory_type, allowed_access, - allowed_usage, data, data_allocator, out_buffer_view); + allocator, shape, shape_rank, element_type, encoding_type, memory_type, + allowed_access, allowed_usage, data, data_allocator, out_buffer_view); } else { return iree_hal_buffer_view_clone_heap_buffer( - allocator, shape, shape_rank, element_type, memory_type, allowed_usage, - iree_make_const_byte_span(data.data, data.data_length), + allocator, shape, shape_rank, element_type, encoding_type, memory_type, + allowed_usage, iree_make_const_byte_span(data.data, data.data_length), out_buffer_view); } } @@ -316,6 +325,12 @@ return iree_hal_element_byte_count(buffer_view->element_type); } +IREE_API_EXPORT iree_hal_encoding_type_t +iree_hal_buffer_view_encoding_type(const iree_hal_buffer_view_t* buffer_view) { + IREE_ASSERT_ARGUMENT(buffer_view); + return buffer_view->encoding_type; +} + IREE_API_EXPORT iree_device_size_t iree_hal_buffer_view_byte_length(const iree_hal_buffer_view_t* buffer_view) { IREE_ASSERT_ARGUMENT(buffer_view); @@ -328,7 +343,7 @@ IREE_ASSERT_ARGUMENT(buffer_view); return iree_hal_buffer_compute_view_offset( buffer_view->shape, buffer_view->shape_rank, buffer_view->element_type, - indices, indices_count, out_offset); + buffer_view->encoding_type, indices, indices_count, out_offset); } IREE_API_EXPORT iree_status_t iree_hal_buffer_view_compute_range( @@ -339,33 +354,51 @@ IREE_ASSERT_ARGUMENT(buffer_view); return iree_hal_buffer_compute_view_range( buffer_view->shape, buffer_view->shape_rank, buffer_view->element_type, - start_indices, indices_count, lengths, lengths_count, out_start_offset, - out_length); + buffer_view->encoding_type, start_indices, indices_count, lengths, + lengths_count, out_start_offset, out_length); } IREE_API_EXPORT iree_status_t iree_hal_buffer_compute_view_size( const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, + iree_hal_encoding_type_t encoding_type, iree_device_size_t* out_allocation_size) { IREE_ASSERT_ARGUMENT(shape); IREE_ASSERT_ARGUMENT(out_allocation_size); *out_allocation_size = 0; - iree_device_size_t byte_length = iree_hal_element_byte_count(element_type); - for (iree_host_size_t i = 0; i < shape_rank; ++i) { - byte_length *= shape[i]; + + iree_device_size_t byte_length = 0; + + switch (encoding_type) { + case IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR: + byte_length = iree_hal_element_byte_count(element_type); + for (iree_host_size_t i = 0; i < shape_rank; ++i) { + byte_length *= shape[i]; + } + break; + default: + return iree_make_status(IREE_STATUS_UNIMPLEMENTED, + "unimplemented encoding type size calculation"); } + *out_allocation_size = byte_length; return iree_ok_status(); } IREE_API_EXPORT iree_status_t iree_hal_buffer_compute_view_offset( const iree_hal_dim_t* shape, iree_host_size_t shape_rank, - iree_hal_element_type_t element_type, const iree_hal_dim_t* indices, + iree_hal_element_type_t element_type, + iree_hal_encoding_type_t encoding_type, const iree_hal_dim_t* indices, iree_host_size_t indices_count, iree_device_size_t* out_offset) { IREE_ASSERT_ARGUMENT(shape); IREE_ASSERT_ARGUMENT(indices); IREE_ASSERT_ARGUMENT(out_offset); *out_offset = 0; + if (encoding_type != IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "only dense encodings support view range computation"); + } if (IREE_UNLIKELY(shape_rank != indices_count)) { return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, "shape rank/indices mismatch: %zu != %zu", @@ -393,7 +426,8 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_compute_view_range( const iree_hal_dim_t* shape, iree_host_size_t shape_rank, - iree_hal_element_type_t element_type, const iree_hal_dim_t* start_indices, + iree_hal_element_type_t element_type, + iree_hal_encoding_type_t encoding_type, const iree_hal_dim_t* start_indices, iree_host_size_t indices_count, const iree_hal_dim_t* lengths, iree_host_size_t lengths_count, iree_device_size_t* out_start_offset, iree_device_size_t* out_length) { @@ -404,6 +438,11 @@ IREE_ASSERT_ARGUMENT(out_length); *out_start_offset = 0; *out_length = 0; + if (encoding_type != IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR) { + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "only dense encodings support view range computation"); + } if (IREE_UNLIKELY(indices_count != lengths_count)) { return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, "indices/lengths mismatch: %zu != %zu", @@ -426,11 +465,11 @@ iree_device_size_t start_byte_offset = 0; IREE_RETURN_IF_ERROR(iree_hal_buffer_compute_view_offset( - shape, shape_rank, element_type, start_indices, indices_count, - &start_byte_offset)); + shape, shape_rank, element_type, encoding_type, start_indices, + indices_count, &start_byte_offset)); iree_device_size_t end_byte_offset = 0; IREE_RETURN_IF_ERROR(iree_hal_buffer_compute_view_offset( - shape, shape_rank, element_type, end_indices, shape_rank, + shape, shape_rank, element_type, encoding_type, end_indices, shape_rank, &end_byte_offset)); // Non-contiguous regions not yet implemented. Will be easier to detect when @@ -482,10 +521,6 @@ IREE_STRING_VIEW_NPOS); } - // f32, i32, etc - iree_hal_element_type_t element_type = IREE_HAL_ELEMENT_TYPE_NONE; - IREE_RETURN_IF_ERROR(iree_hal_parse_element_type(type_str, &element_type)); - // AxBxC... iree_host_size_t shape_rank = 0; iree_status_t shape_result = @@ -504,10 +539,18 @@ IREE_RETURN_IF_ERROR( iree_hal_parse_shape(shape_str, shape_rank, shape, &shape_rank)); + // f32, i32, etc + iree_hal_element_type_t element_type = IREE_HAL_ELEMENT_TYPE_NONE; + IREE_RETURN_IF_ERROR(iree_hal_parse_element_type(type_str, &element_type)); + + // TODO(benvanik): allow specifying the encoding. + iree_hal_encoding_type_t encoding_type = + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR; + // Allocate the buffer we will parse into from the provided allocator. iree_device_size_t buffer_length = 0; IREE_RETURN_IF_ERROR(iree_hal_buffer_compute_view_size( - shape, shape_rank, element_type, &buffer_length)); + shape, shape_rank, element_type, encoding_type, &buffer_length)); iree_hal_buffer_t* buffer = NULL; IREE_RETURN_IF_ERROR(iree_hal_allocator_allocate_buffer( buffer_allocator, @@ -535,7 +578,7 @@ // Wrap and pass ownership of the buffer to the buffer view. status = iree_hal_buffer_view_create(buffer, shape, shape_rank, element_type, - out_buffer_view); + encoding_type, out_buffer_view); iree_hal_buffer_release(buffer); return status; } @@ -612,6 +655,8 @@ return status; } + // TODO(benvanik): allow printing the encoding. + // Separator: <meta>=<value> APPEND_CHAR('=');
diff --git a/iree/hal/buffer_view.h b/iree/hal/buffer_view.h index a4595e6..310a487 100644 --- a/iree/hal/buffer_view.h +++ b/iree/hal/buffer_view.h
@@ -77,6 +77,23 @@ typedef uint32_t iree_hal_element_type_t; // clang-format on +// Defines the encoding type of a buffer when known. +enum iree_hal_encoding_types_t { + // Encoding is unknown or unspecified. Generic interpretation of the buffer + // contents is not possible. + IREE_HAL_ENCODING_TYPE_OPAQUE = 0, + // Encoding is a densely-packed numpy/C-style row-major format. + // All elements are contiguous in memory. + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR = 1, + // TODO(#6762): sparse encodings we care about (_SPARSE_CSR) + // We will likely want to make this a bitfield like the element type is that + // we can more easily distinguish between encoding types that we can use for + // certain operations; for example, size calculations on a DENSE_ROW_MAJOR + // and DENSE_COLUMN_MAJOR would be easier to perform if we had a bit to test + // for whether it's dense. +}; +typedef uint32_t iree_hal_encoding_type_t; + // A dimension within a shape. typedef int32_t iree_hal_dim_t; @@ -88,18 +105,23 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_compute_view_size( const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, + iree_hal_encoding_type_t encoding_type, iree_device_size_t* out_allocation_size); // Calculates a byte offset into a buffer at the given indices. +// Only works with densely-packed representations. IREE_API_EXPORT iree_status_t iree_hal_buffer_compute_view_offset( const iree_hal_dim_t* shape, iree_host_size_t shape_rank, - iree_hal_element_type_t element_type, const iree_hal_dim_t* indices, + iree_hal_element_type_t element_type, + iree_hal_encoding_type_t encoding_type, const iree_hal_dim_t* indices, size_t indices_count, iree_device_size_t* out_offset); // Calculates a byte range into a buffer of the given contiguous range. +// Only works with densely-packed representations. IREE_API_EXPORT iree_status_t iree_hal_buffer_compute_view_range( const iree_hal_dim_t* shape, iree_host_size_t shape_rank, - iree_hal_element_type_t element_type, const iree_hal_dim_t* start_indices, + iree_hal_element_type_t element_type, + iree_hal_encoding_type_t encoding_type, const iree_hal_dim_t* start_indices, iree_host_size_t indices_count, const iree_hal_dim_t* lengths, iree_host_size_t lengths_count, iree_device_size_t* out_start_offset, iree_device_size_t* out_length); @@ -121,6 +143,7 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_create( iree_hal_buffer_t* buffer, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, + iree_hal_encoding_type_t encoding_type, iree_hal_buffer_view_t** out_buffer_view); // Allocates a buffer from |allocator| and wraps it in a buffer view. @@ -131,7 +154,8 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_allocate_buffer( iree_hal_allocator_t* allocator, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, - iree_hal_memory_type_t memory_type, iree_hal_buffer_usage_t allowed_usage, + iree_hal_encoding_type_t encoding_type, iree_hal_memory_type_t memory_type, + iree_hal_buffer_usage_t allowed_usage, iree_hal_buffer_view_t** out_buffer_view); // Clones a host buffer using |allocator| and wraps it in a buffer view. @@ -146,8 +170,9 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_clone_heap_buffer( iree_hal_allocator_t* allocator, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, - iree_hal_memory_type_t memory_type, iree_hal_buffer_usage_t allowed_usage, - iree_const_byte_span_t data, iree_hal_buffer_view_t** out_buffer_view); + iree_hal_encoding_type_t encoding_type, iree_hal_memory_type_t memory_type, + iree_hal_buffer_usage_t allowed_usage, iree_const_byte_span_t data, + iree_hal_buffer_view_t** out_buffer_view); // Imports a host buffer using |allocator| and wraps it in a buffer view. // This is equivalent to: @@ -160,7 +185,8 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_wrap_heap_buffer( iree_hal_allocator_t* allocator, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, - iree_hal_memory_type_t memory_type, iree_hal_memory_access_t allowed_access, + iree_hal_encoding_type_t encoding_type, iree_hal_memory_type_t memory_type, + iree_hal_memory_access_t allowed_access, iree_hal_buffer_usage_t allowed_usage, iree_byte_span_t data, iree_allocator_t data_allocator, iree_hal_buffer_view_t** out_buffer_view); @@ -178,7 +204,8 @@ IREE_API_EXPORT iree_status_t iree_hal_buffer_view_wrap_or_clone_heap_buffer( iree_hal_allocator_t* allocator, const iree_hal_dim_t* shape, iree_host_size_t shape_rank, iree_hal_element_type_t element_type, - iree_hal_memory_type_t memory_type, iree_hal_memory_access_t allowed_access, + iree_hal_encoding_type_t encoding_type, iree_hal_memory_type_t memory_type, + iree_hal_memory_access_t allowed_access, iree_hal_buffer_usage_t allowed_usage, iree_byte_span_t data, iree_allocator_t data_allocator, iree_hal_buffer_view_t** out_buffer_view); @@ -243,6 +270,10 @@ IREE_API_EXPORT iree_host_size_t iree_hal_buffer_view_element_size(const iree_hal_buffer_view_t* buffer_view); +// Returns the encoding type of the buffer. +IREE_API_EXPORT iree_hal_encoding_type_t +iree_hal_buffer_view_encoding_type(const iree_hal_buffer_view_t* buffer_view); + // Returns the total size of the specified view in bytes. // Note that not all buffers are contiguous or densely packed. IREE_API_EXPORT iree_device_size_t
diff --git a/iree/hal/string_util_test.cc b/iree/hal/string_util_test.cc index 968f66e..c661520 100644 --- a/iree/hal/string_util_test.cc +++ b/iree/hal/string_util_test.cc
@@ -427,9 +427,12 @@ static StatusOr<BufferView> Create(Buffer buffer, iree::span<const iree_hal_dim_t> shape, iree_hal_element_type_t element_type) { + iree_hal_encoding_type_t encoding_type = + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR; BufferView buffer_view; - iree_status_t status = iree_hal_buffer_view_create( - buffer, shape.data(), shape.size(), element_type, &buffer_view); + iree_status_t status = + iree_hal_buffer_view_create(buffer, shape.data(), shape.size(), + element_type, encoding_type, &buffer_view); IREE_RETURN_IF_ERROR(std::move(status)); return std::move(buffer_view); }
diff --git a/iree/modules/check/check_test.cc b/iree/modules/check/check_test.cc index caafb2c..dbf03c0 100644 --- a/iree/modules/check/check_test.cc +++ b/iree/modules/check/check_test.cc
@@ -91,7 +91,7 @@ buffer.get(), 0, contents.data(), contents.size() * sizeof(int32_t))); IREE_ASSERT_OK(iree_hal_buffer_view_create( buffer.get(), shape.data(), shape.size(), IREE_HAL_ELEMENT_TYPE_SINT_32, - &*out_buffer_view)); + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, &*out_buffer_view)); } void CreateFloat16BufferView(iree::span<const uint16_t> contents, @@ -114,7 +114,8 @@ buffer.get(), 0, contents.data(), contents.size() * sizeof(uint16_t))); IREE_ASSERT_OK(iree_hal_buffer_view_create( buffer.get(), shape.data(), shape.size(), - IREE_HAL_ELEMENT_TYPE_FLOAT_16, &*out_buffer_view)); + IREE_HAL_ELEMENT_TYPE_FLOAT_16, IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, + &*out_buffer_view)); } void CreateFloat32BufferView(iree::span<const float> contents, @@ -136,7 +137,8 @@ contents.size() * sizeof(float))); IREE_ASSERT_OK(iree_hal_buffer_view_create( buffer.get(), shape.data(), shape.size(), - IREE_HAL_ELEMENT_TYPE_FLOAT_32, &*out_buffer_view)); + IREE_HAL_ELEMENT_TYPE_FLOAT_32, IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, + &*out_buffer_view)); } void CreateFloat64BufferView(iree::span<const double> contents, @@ -158,7 +160,8 @@ buffer.get(), 0, contents.data(), contents.size() * sizeof(double))); IREE_ASSERT_OK(iree_hal_buffer_view_create( buffer.get(), shape.data(), shape.size(), - IREE_HAL_ELEMENT_TYPE_FLOAT_64, &*out_buffer_view)); + IREE_HAL_ELEMENT_TYPE_FLOAT_64, IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, + &*out_buffer_view)); } iree_status_t Invoke(const char* function_name) {
diff --git a/iree/modules/hal/module.c b/iree/modules/hal/module.c index 14924d9..3e2edc0 100644 --- a/iree/modules/hal/module.c +++ b/iree/modules/hal/module.c
@@ -404,14 +404,18 @@ iree_hal_buffer_t* source_buffer = NULL; IREE_RETURN_IF_ERROR(iree_hal_buffer_check_deref(args->r0, &source_buffer)); iree_hal_element_type_t element_type = (iree_hal_element_type_t)args->i1; + // TODO(#6762): add encoding type to vm.import. + iree_hal_encoding_type_t encoding_type = + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR; iree_host_size_t shape_rank = 0; iree_hal_dim_t* shape_dims = NULL; IREE_VM_ABI_VLA_STACK_CAST(args, a2_count, a2, iree_hal_dim_t, 128, &shape_rank, &shape_dims); iree_hal_buffer_view_t* buffer_view = NULL; - IREE_RETURN_IF_ERROR(iree_hal_buffer_view_create( - source_buffer, shape_dims, shape_rank, element_type, &buffer_view)); + IREE_RETURN_IF_ERROR( + iree_hal_buffer_view_create(source_buffer, shape_dims, shape_rank, + element_type, encoding_type, &buffer_view)); rets->r0 = iree_hal_buffer_view_move_ref(buffer_view); return iree_ok_status(); }
diff --git a/iree/runtime/demo/hello_world_explained.c b/iree/runtime/demo/hello_world_explained.c index fb4f7e2..cbdfb09 100644 --- a/iree/runtime/demo/hello_world_explained.c +++ b/iree/runtime/demo/hello_world_explained.c
@@ -196,6 +196,8 @@ arg0_shape, IREE_ARRAYSIZE(arg0_shape), // Element type: IREE_HAL_ELEMENT_TYPE_FLOAT_32, + // Encoding type: + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, // Where to allocate (host or device): IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, // What access to allow to this memory (this is .rodata so READ only): @@ -227,6 +229,7 @@ status = iree_hal_buffer_view_wrap_or_clone_heap_buffer( device_allocator, arg1_shape, IREE_ARRAYSIZE(arg1_shape), IREE_HAL_ELEMENT_TYPE_FLOAT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_MEMORY_ACCESS_READ, IREE_HAL_BUFFER_USAGE_ALL, iree_make_byte_span((void*)arg1_data, sizeof(arg1_data)),
diff --git a/iree/runtime/demo/hello_world_terse.c b/iree/runtime/demo/hello_world_terse.c index 2a71bb8..3a10d32 100644 --- a/iree/runtime/demo/hello_world_terse.c +++ b/iree/runtime/demo/hello_world_terse.c
@@ -82,6 +82,7 @@ IREE_CHECK_OK(iree_hal_buffer_view_wrap_or_clone_heap_buffer( iree_runtime_session_device_allocator(session), arg0_shape, IREE_ARRAYSIZE(arg0_shape), IREE_HAL_ELEMENT_TYPE_FLOAT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_MEMORY_ACCESS_READ, IREE_HAL_BUFFER_USAGE_ALL, iree_make_byte_span((void*)arg0_data, sizeof(arg0_data)), @@ -100,6 +101,7 @@ IREE_CHECK_OK(iree_hal_buffer_view_wrap_or_clone_heap_buffer( iree_runtime_session_device_allocator(session), arg1_shape, IREE_ARRAYSIZE(arg1_shape), IREE_HAL_ELEMENT_TYPE_FLOAT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_MEMORY_ACCESS_READ, IREE_HAL_BUFFER_USAGE_ALL, iree_make_byte_span((void*)arg1_data, sizeof(arg1_data)),
diff --git a/iree/samples/custom_modules/custom_modules_test.cc b/iree/samples/custom_modules/custom_modules_test.cc index 1dc2052..10f9b15 100644 --- a/iree/samples/custom_modules/custom_modules_test.cc +++ b/iree/samples/custom_modules/custom_modules_test.cc
@@ -135,7 +135,7 @@ iree_hal_buffer_view_t* buffer_view = nullptr; IREE_ASSERT_OK(iree_hal_buffer_view_wrap_or_clone_heap_buffer( hal_allocator_, kShape, IREE_ARRAYSIZE(kShape), - IREE_HAL_ELEMENT_TYPE_FLOAT_32, + IREE_HAL_ELEMENT_TYPE_FLOAT_32, IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_MEMORY_ACCESS_ALL, IREE_HAL_BUFFER_USAGE_ALL, iree_make_byte_span((void*)kBufferContents, sizeof(kBufferContents)), @@ -179,7 +179,7 @@ iree_hal_buffer_view_t* buffer_view = nullptr; IREE_ASSERT_OK(iree_hal_buffer_view_wrap_or_clone_heap_buffer( hal_allocator_, kShape, IREE_ARRAYSIZE(kShape), - IREE_HAL_ELEMENT_TYPE_FLOAT_32, + IREE_HAL_ELEMENT_TYPE_FLOAT_32, IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_MEMORY_ACCESS_ALL, IREE_HAL_BUFFER_USAGE_ALL, iree_make_byte_span((void*)kBufferContents, sizeof(kBufferContents)),
diff --git a/iree/samples/dynamic_shapes/main.c b/iree/samples/dynamic_shapes/main.c index 1e1838d..f82737c 100644 --- a/iree/samples/dynamic_shapes/main.c +++ b/iree/samples/dynamic_shapes/main.c
@@ -24,6 +24,7 @@ status = iree_hal_buffer_view_clone_heap_buffer( iree_runtime_session_device_allocator(session), arg0_shape, IREE_ARRAYSIZE(arg0_shape), IREE_HAL_ELEMENT_TYPE_SINT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_BUFFER_USAGE_ALL, iree_make_const_byte_span((void*)values, sizeof(int) * values_length), @@ -75,6 +76,7 @@ status = iree_hal_buffer_view_clone_heap_buffer( iree_runtime_session_device_allocator(session), arg0_shape, IREE_ARRAYSIZE(arg0_shape), IREE_HAL_ELEMENT_TYPE_SINT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_BUFFER_USAGE_ALL, iree_make_const_byte_span((void*)values, sizeof(int) * values_length), @@ -114,6 +116,7 @@ status = iree_hal_buffer_view_clone_heap_buffer( iree_runtime_session_device_allocator(session), arg0_shape, IREE_ARRAYSIZE(arg0_shape), IREE_HAL_ELEMENT_TYPE_SINT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_BUFFER_USAGE_ALL, iree_make_const_byte_span((void*)values, sizeof(int) * values_length),
diff --git a/iree/samples/simple_embedding/simple_embedding.c b/iree/samples/simple_embedding/simple_embedding.c index 2f5645d..c04a15e 100644 --- a/iree/samples/simple_embedding/simple_embedding.c +++ b/iree/samples/simple_embedding/simple_embedding.c
@@ -92,10 +92,10 @@ iree_hal_buffer_view_t* arg1_buffer_view = NULL; IREE_RETURN_IF_ERROR(iree_hal_buffer_view_create( arg0_buffer, shape, IREE_ARRAYSIZE(shape), IREE_HAL_ELEMENT_TYPE_FLOAT_32, - &arg0_buffer_view)); + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, &arg0_buffer_view)); IREE_RETURN_IF_ERROR(iree_hal_buffer_view_create( arg1_buffer, shape, IREE_ARRAYSIZE(shape), IREE_HAL_ELEMENT_TYPE_FLOAT_32, - &arg1_buffer_view)); + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, &arg1_buffer_view)); iree_hal_buffer_release(arg0_buffer); iree_hal_buffer_release(arg1_buffer);
diff --git a/iree/samples/static_library/static_library_demo.c b/iree/samples/static_library/static_library_demo.c index 47b32c2..350edac 100644 --- a/iree/samples/static_library/static_library_demo.c +++ b/iree/samples/static_library/static_library_demo.c
@@ -133,8 +133,8 @@ if (iree_status_is_ok(status)) { iree_hal_buffer_view_clone_heap_buffer( iree_hal_device_allocator(device), shape, IREE_ARRAYSIZE(shape), - IREE_HAL_ELEMENT_TYPE_FLOAT_32, input_memory_type, - IREE_HAL_BUFFER_USAGE_ALL, + IREE_HAL_ELEMENT_TYPE_FLOAT_32, IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, + input_memory_type, IREE_HAL_BUFFER_USAGE_ALL, iree_make_const_byte_span((void*)kFloat4, sizeof(float) * kElementCount), &arg0_buffer_view); @@ -142,8 +142,8 @@ if (iree_status_is_ok(status)) { iree_hal_buffer_view_clone_heap_buffer( iree_hal_device_allocator(device), shape, IREE_ARRAYSIZE(shape), - IREE_HAL_ELEMENT_TYPE_FLOAT_32, input_memory_type, - IREE_HAL_BUFFER_USAGE_ALL, + IREE_HAL_ELEMENT_TYPE_FLOAT_32, IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, + input_memory_type, IREE_HAL_BUFFER_USAGE_ALL, iree_make_const_byte_span((void*)kFloat2, sizeof(float) * kElementCount), &arg1_buffer_view);
diff --git a/iree/samples/variables_and_state/main.c b/iree/samples/variables_and_state/main.c index c2fe84d..5abb8a0 100644 --- a/iree/samples/variables_and_state/main.c +++ b/iree/samples/variables_and_state/main.c
@@ -56,6 +56,7 @@ status = iree_hal_buffer_view_clone_heap_buffer( iree_runtime_session_device_allocator(session), arg0_shape, IREE_ARRAYSIZE(arg0_shape), IREE_HAL_ELEMENT_TYPE_SINT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_BUFFER_USAGE_ALL, iree_make_const_byte_span((void*)arg0_data, sizeof(arg0_data)), &arg0); @@ -88,6 +89,7 @@ status = iree_hal_buffer_view_clone_heap_buffer( iree_runtime_session_device_allocator(session), arg0_shape, IREE_ARRAYSIZE(arg0_shape), IREE_HAL_ELEMENT_TYPE_SINT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_BUFFER_USAGE_ALL, iree_make_const_byte_span((void*)arg0_data, sizeof(arg0_data)), &arg0);
diff --git a/iree/samples/vulkan/vulkan_inference_gui.cc b/iree/samples/vulkan/vulkan_inference_gui.cc index c688a9f..f30b810 100644 --- a/iree/samples/vulkan/vulkan_inference_gui.cc +++ b/iree/samples/vulkan/vulkan_inference_gui.cc
@@ -388,11 +388,13 @@ IREE_CHECK_OK(iree_hal_buffer_view_create( input0_buffer, /*shape=*/&kElementCount, /*shape_rank=*/1, - IREE_HAL_ELEMENT_TYPE_FLOAT_32, &input0_buffer_view)); + IREE_HAL_ELEMENT_TYPE_FLOAT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, &input0_buffer_view)); IREE_CHECK_OK(iree_hal_buffer_view_create( input1_buffer, /*shape=*/&kElementCount, /*shape_rank=*/1, - IREE_HAL_ELEMENT_TYPE_FLOAT_32, &input1_buffer_view)); + IREE_HAL_ELEMENT_TYPE_FLOAT_32, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, &input1_buffer_view)); iree_hal_buffer_release(input0_buffer); iree_hal_buffer_release(input1_buffer); // Marshal inputs through a VM variant list.
diff --git a/iree/tools/utils/image_util.c b/iree/tools/utils/image_util.c index 6e9440c..60f7708 100644 --- a/iree/tools/utils/image_util.c +++ b/iree/tools/utils/image_util.c
@@ -136,6 +136,7 @@ // SINT_8 and UINT_8 perform direct buffer wrap. result = iree_hal_buffer_view_wrap_or_clone_heap_buffer( allocator, shape, shape_rank, element_type, + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, IREE_HAL_MEMORY_TYPE_HOST_LOCAL | IREE_HAL_MEMORY_TYPE_DEVICE_VISIBLE, IREE_HAL_MEMORY_ACCESS_READ, IREE_HAL_BUFFER_USAGE_ALL, iree_make_byte_span((void*)pixel_data, element_byte * buffer_length), @@ -186,8 +187,11 @@ iree_hal_buffer_unmap_range(&mapped_memory); } if (iree_status_is_ok(result)) { - result = iree_hal_buffer_view_create(buffer, shape, shape_rank, - element_type, out_buffer_view); + iree_hal_encoding_type_t encoding_type = + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR; + result = + iree_hal_buffer_view_create(buffer, shape, shape_rank, element_type, + encoding_type, out_buffer_view); } iree_hal_buffer_release(buffer); stbi_image_free(pixel_data);
diff --git a/iree/tools/utils/trace_replay.c b/iree/tools/utils/trace_replay.c index faa8601..0830a13 100644 --- a/iree/tools/utils/trace_replay.c +++ b/iree/tools/utils/trace_replay.c
@@ -417,6 +417,42 @@ return iree_hal_parse_element_type(element_type_str, out_element_type); } +// Parses an encoding type. +// +// ```yaml +// encoding_type: 50331680 +// ``` +static iree_status_t iree_trace_replay_parse_hal_encoding_type( + iree_trace_replay_t* replay, yaml_document_t* document, + yaml_node_t* encoding_type_node, + iree_hal_encoding_type_t* out_encoding_type) { + *out_encoding_type = IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR; + + iree_string_view_t encoding_type_str = + iree_yaml_node_as_string(encoding_type_node); + if (iree_string_view_is_empty(encoding_type_str)) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "(%zu): encoding type missing", + encoding_type_node->start_mark.line); + } + + // If the first character is a digit then interpret as a %d type. + if (isdigit(encoding_type_str.data[0])) { + static_assert(sizeof(*out_encoding_type) == sizeof(uint32_t), "4 bytes"); + if (!iree_string_view_atoi_uint32(encoding_type_str, out_encoding_type)) { + return iree_make_status(IREE_STATUS_OUT_OF_RANGE, + "(%zu): invalid encoding type", + encoding_type_node->start_mark.line); + } + return iree_ok_status(); + } + + // Parse as a canonical encoding type. + // TODO(#6762): implement iree_hal_parse_element_type. + return iree_make_status(IREE_STATUS_UNIMPLEMENTED, + "iree_hal_parse_encoding_type not implemented"); +} + // Parses a serialized !hal.buffer into |buffer|. // // ```yaml @@ -469,14 +505,6 @@ static iree_status_t iree_trace_replay_parse_hal_buffer_view( iree_trace_replay_t* replay, yaml_document_t* document, yaml_node_t* value_node, iree_vm_list_t* target_list) { - yaml_node_t* element_type_node = NULL; - IREE_RETURN_IF_ERROR(iree_yaml_mapping_find( - document, value_node, iree_make_cstring_view("element_type"), - &element_type_node)); - iree_hal_element_type_t element_type = IREE_HAL_ELEMENT_TYPE_NONE; - IREE_RETURN_IF_ERROR(iree_trace_replay_parse_hal_element_type( - replay, document, element_type_node, &element_type)); - yaml_node_t* shape_node = NULL; IREE_RETURN_IF_ERROR(iree_yaml_mapping_try_find( document, value_node, iree_make_cstring_view("shape"), &shape_node)); @@ -485,6 +513,23 @@ IREE_RETURN_IF_ERROR(iree_trace_replay_parse_hal_shape( replay, document, shape_node, IREE_ARRAYSIZE(shape), shape, &shape_rank)); + yaml_node_t* element_type_node = NULL; + IREE_RETURN_IF_ERROR(iree_yaml_mapping_find( + document, value_node, iree_make_cstring_view("element_type"), + &element_type_node)); + iree_hal_element_type_t element_type = IREE_HAL_ELEMENT_TYPE_NONE; + IREE_RETURN_IF_ERROR(iree_trace_replay_parse_hal_element_type( + replay, document, element_type_node, &element_type)); + + yaml_node_t* encoding_type_node = NULL; + IREE_RETURN_IF_ERROR(iree_yaml_mapping_try_find( + document, value_node, iree_make_cstring_view("encoding_type"), + &encoding_type_node)); + iree_hal_encoding_type_t encoding_type = + IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR; + IREE_RETURN_IF_ERROR(iree_trace_replay_parse_hal_encoding_type( + replay, document, encoding_type_node, &encoding_type)); + yaml_node_t* contents_node = NULL; IREE_RETURN_IF_ERROR(iree_yaml_mapping_try_find( document, value_node, iree_make_cstring_view("contents"), @@ -492,7 +537,8 @@ iree_device_size_t allocation_size = 0; IREE_RETURN_IF_ERROR(iree_hal_buffer_compute_view_size( - shape, shape_rank, element_type, &allocation_size)); + shape, shape_rank, element_type, IREE_HAL_ENCODING_TYPE_DENSE_ROW_MAJOR, + &allocation_size)); iree_hal_buffer_t* buffer = NULL; IREE_RETURN_IF_ERROR(iree_hal_allocator_allocate_buffer( @@ -508,7 +554,7 @@ iree_hal_buffer_view_t* buffer_view = NULL; status = iree_hal_buffer_view_create(buffer, shape, shape_rank, element_type, - &buffer_view); + encoding_type, &buffer_view); iree_hal_buffer_release(buffer); IREE_RETURN_IF_ERROR(status);