Add support for inline HAL This CL adds support for inline HAL bytecode (both vmvx and llvm-static-library). EmitC is currently not supported at IREE upstream. As Inline HAL is not stable, currently we only generate inline HAL targets for samples/simple_vec_mul. Change-Id: I6058c49dea8b3486c183fd500f532768640cf0a3
diff --git a/cmake/springbok_modules.cmake b/cmake/springbok_modules.cmake index 5f511c7..a221f71 100644 --- a/cmake/springbok_modules.cmake +++ b/cmake/springbok_modules.cmake
@@ -9,6 +9,7 @@ # If omitted then no C embed code will be generated. # RVV_OFF: Indicate RVV is OFF (default: ON) # VMVX: Compile VMVX backend +# INLINE_HAL: Use inline HAL. # # Examples: # springbok_modules( @@ -39,7 +40,7 @@ function(springbok_modules) cmake_parse_arguments( _RULE - "PUBLIC;RVV_OFF;VMVX" + "PUBLIC;RVV_OFF;VMVX;INLINE_HAL" "NAME;SRC;C_IDENTIFIER" "FLAGS" ${ARGN} @@ -49,6 +50,10 @@ set(_RVV_OFF_ARG "RVV_OFF") endif() + if (${_RULE_INLINE_HAL}) + set(_INLINE_HAL_ARG "INLINE_HAL") + endif() + springbok_static_module( NAME "${_RULE_NAME}_bytecode_module_static" @@ -59,6 +64,7 @@ FLAGS "${_RULE_FLAGS}" "${_RVV_OFF_ARG}" + "${_INLINE_HAL_ARG}" ) springbok_static_module( @@ -82,6 +88,7 @@ "${_RULE_C_IDENTIFIER}_bytecode_module_vmvx" FLAGS "${_RULE_FLAGS}" + "${_INLINE_HAL_ARG}" ) springbok_vmvx_module(
diff --git a/cmake/springbok_static_module.cmake b/cmake/springbok_static_module.cmake index e30e54a..06a0aed 100644 --- a/cmake/springbok_static_module.cmake +++ b/cmake/springbok_static_module.cmake
@@ -8,6 +8,7 @@ # FLAGS: Flags to pass to the translation tool (list of strings). # RVV_OFF: Indicate RVV is OFF (default: ON) # EMITC: Uses EmitC to output C code instead of VM bytecode. +# INLINE_HAL: Use inline HAL. # # Examples: # springbok_static_module( @@ -37,7 +38,7 @@ function(springbok_static_module) cmake_parse_arguments( _RULE - "RVV_OFF;EMITC" + "RVV_OFF;EMITC;INLINE_HAL" "NAME;SRC;C_IDENTIFIER" "FLAGS" ${ARGN} @@ -86,6 +87,9 @@ list(APPEND _COMPILER_ARGS "--iree-llvm-target-cpu-features=${_CPU_FEATURES}") list(APPEND _COMPILER_ARGS "--iree-llvm-target-abi=ilp32") list(APPEND _COMPILER_ARGS "--iree-llvm-link-embedded=false") + if (${_RULE_INLINE_HAL}) + list(APPEND _COMPILER_ARGS "--iree-execution-model=inline-dynamic") + endif() if(_RULE_EMITC) # TODO(b/245584726): iree_c_module causes long build time for some models
diff --git a/cmake/springbok_vmvx_module.cmake b/cmake/springbok_vmvx_module.cmake index 8fe916b..687b3ef 100644 --- a/cmake/springbok_vmvx_module.cmake +++ b/cmake/springbok_vmvx_module.cmake
@@ -6,6 +6,7 @@ # SRC: Source file to compile into a bytecode module. Support relative path. # FLAGS: Flags to pass to the translation tool (list of strings). # EMITC: Uses EmitC to output C code instead of VM bytecode. +# INLINE_HAL: Use inline HAL. # # Examples: # springbok_vmvx_module( @@ -34,7 +35,7 @@ function(springbok_vmvx_module) cmake_parse_arguments( _RULE - "EMITC" + "EMITC;INLINE_HAL" "NAME;SRC;C_IDENTIFIER" "FLAGS" ${ARGN} @@ -69,7 +70,12 @@ # Set common iree-compile flags set(_COMPILER_ARGS ${_RULE_FLAGS}) - list(APPEND _COMPILER_ARGS "--iree-hal-target-backends=vmvx") + if (${_RULE_INLINE_HAL}) + list(APPEND _COMPILER_ARGS "--iree-execution-model=inline-static") + list(APPEND _COMPILER_ARGS "--iree-hal-target-backends=vmvx-inline") + else() + list(APPEND _COMPILER_ARGS "--iree-hal-target-backends=vmvx") + endif() if(_RULE_EMITC) list(APPEND _COMPILER_ARGS "--iree-vm-target-index-bits=32")
diff --git a/device/device.h b/device/device.h index cfd172d..2158ca3 100644 --- a/device/device.h +++ b/device/device.h
@@ -19,11 +19,13 @@ #include "iree/base/api.h" #include "iree/hal/api.h" +#include "iree/hal/local/executable_loader.h" // Create the HAL device from the different backend targets. -// The HAL device is returned based on the implementation, and it must be -// released by the caller. +// The HAL device and loader are returned based on the implementation, and they +// must be released by the caller. iree_status_t create_sample_device(iree_allocator_t host_allocator, - iree_hal_device_t** out_device); + iree_hal_device_t** out_device, + iree_hal_executable_loader_t** loader); #endif // DEVICE_DEVICE_H_
diff --git a/device/device_static_loader.c b/device/device_static_loader.c index 22c8f36..35b6f31 100644 --- a/device/device_static_loader.c +++ b/device/device_static_loader.c
@@ -23,10 +23,11 @@ #include "model_util/model_api.h" // A function to create the HAL device from the different backend targets. -// The HAL device is returned based on the implementation, and it must be -// released by the caller. +// The HAL device and loader are returned based on the implementation, and they +// must be released by the caller. iree_status_t create_sample_device(iree_allocator_t host_allocator, - iree_hal_device_t** out_device) { + iree_hal_device_t** out_device, + iree_hal_executable_loader_t** loader) { iree_status_t status = iree_ok_status(); // Set paramters for the device created in the next step. @@ -36,12 +37,10 @@ // Load the statically embedded library const iree_hal_executable_library_query_fn_t libraries[] = {library_query()}; - iree_hal_executable_loader_t* library_loader = NULL; if (iree_status_is_ok(status)) { status = iree_hal_static_library_loader_create( IREE_ARRAYSIZE(libraries), libraries, - iree_hal_executable_import_provider_null(), host_allocator, - &library_loader); + iree_hal_executable_import_provider_null(), host_allocator, loader); } // Use the default host allocator for buffer allocations. @@ -55,11 +54,10 @@ // Create the device and release the executor and loader afterwards. if (iree_status_is_ok(status)) { status = iree_hal_sync_device_create( - identifier, ¶ms, /*loader_count=*/1, &library_loader, - device_allocator, host_allocator, out_device); + identifier, ¶ms, /*loader_count=*/1, loader, device_allocator, + host_allocator, out_device); } iree_hal_allocator_release(device_allocator); - iree_hal_executable_loader_release(library_loader); return status; }
diff --git a/device/device_vmvx_loader.c b/device/device_vmvx_loader.c index e1d89a8..8945489 100644 --- a/device/device_vmvx_loader.c +++ b/device/device_vmvx_loader.c
@@ -23,10 +23,11 @@ #include "model_util/model_api.h" // A function to create the HAL device from the different backend targets. -// The HAL device is returned based on the implementation, and it must be -// released by the caller. +// The HAL device and loader are returned based on the implementation, and they +// must be released by the caller. iree_status_t create_sample_device(iree_allocator_t host_allocator, - iree_hal_device_t** out_device) { + iree_hal_device_t** out_device, + iree_hal_executable_loader_t** loader) { // Set parameters for the device created in the next step. iree_hal_sync_device_params_t params; iree_hal_sync_device_params_initialize(¶ms); @@ -34,11 +35,10 @@ iree_vm_instance_t* instance = NULL; iree_status_t status = iree_vm_instance_create(host_allocator, &instance); - iree_hal_executable_loader_t* loader = NULL; if (iree_status_is_ok(status)) { status = iree_hal_vmvx_module_loader_create( instance, /*user_module_count=*/0, /*user_modules=*/NULL, - host_allocator, &loader); + host_allocator, loader); } iree_vm_instance_release(instance); @@ -53,11 +53,10 @@ if (iree_status_is_ok(status)) { // Create the synchronous device. status = iree_hal_sync_device_create( - identifier, ¶ms, /*loader_count=*/1, &loader, device_allocator, + identifier, ¶ms, /*loader_count=*/1, loader, device_allocator, host_allocator, out_device); } iree_hal_allocator_release(device_allocator); - iree_hal_executable_loader_release(loader); return status; }
diff --git a/model_util/CMakeLists.txt b/model_util/CMakeLists.txt index c14745f..3bbbebc 100644 --- a/model_util/CMakeLists.txt +++ b/model_util/CMakeLists.txt
@@ -16,6 +16,7 @@ iree::vm::shims_emitc ) +# static library using regular HAL iree_cc_library( NAME util_static @@ -24,6 +25,7 @@ device::device_static_loader ) +# vmvx using regular HAL iree_cc_library( NAME util_vmvx @@ -32,6 +34,53 @@ device::device_vmvx_loader ) +# static library using inline HAL +iree_cc_library( + NAME + util_static_inline + HDRS + "util.h" + SRCS + "util.c" + DEPS + ::alloc + device::device_static_loader + iree::base + iree::hal + iree::hal::local + iree::hal::drivers::local_sync::sync_driver + iree::modules::hal + iree::modules::hal::inline + iree::modules::hal::loader + iree::vm + iree::vm::shims_emitc + COPTS + "-DBUILD_LOADER_HAL" +) + +# vmvx using inline HAL +iree_cc_library( + NAME + util_vmvx_inline + HDRS + "util.h" + SRCS + "util.c" + DEPS + ::alloc + device::device_vmvx_loader + iree::base + iree::hal + iree::hal::local + iree::hal::drivers::local_sync::sync_driver + iree::modules::hal + iree::modules::hal::inline + iree::vm + iree::vm::shims_emitc + COPTS + "-DBUILD_INLINE_HAL" +) + iree_cc_library( NAME alloc
diff --git a/model_util/util.c b/model_util/util.c index 1c656f0..24d7744 100644 --- a/model_util/util.c +++ b/model_util/util.c
@@ -24,6 +24,8 @@ #include "device/device.h" #include "iree/base/api.h" #include "iree/hal/api.h" +#include "iree/modules/hal/inline/module.h" +#include "iree/modules/hal/loader/module.h" #include "iree/modules/hal/module.h" #include "iree/vm/api.h" #include "iree/vm/bytecode_module.h" @@ -38,6 +40,82 @@ extern const MlModel kModel; +// Create context that will hold the module state across invocations. +static iree_status_t create_context(iree_vm_instance_t *instance, + iree_hal_device_t **device, + iree_vm_context_t **context) { + iree_allocator_t host_allocator = iree_allocator_system(); + iree_status_t result = iree_vm_instance_create(host_allocator, &instance); + +#if defined(BUILD_INLINE_HAL) + IREE_RETURN_IF_ERROR(iree_hal_module_register_inline_types(instance)); +#elif defined(BUILD_LOADER_HAL) + IREE_RETURN_IF_ERROR(iree_hal_module_register_loader_types(instance)); +#else + IREE_RETURN_IF_ERROR(iree_hal_module_register_all_types(instance)); +#endif + + iree_hal_executable_loader_t *loader = NULL; + if (iree_status_is_ok(result)) { + result = create_sample_device(host_allocator, device, &loader); + } + + // Load bytecode or C module. + iree_vm_module_t *module = NULL; + if (iree_status_is_ok(result)) { + result = create_module(instance, &module); + } + +#if defined(BUILD_INLINE_HAL) || defined(BUILD_LOADER_HAL) + // Create hal_inline_module + iree_vm_module_t *hal_inline_module = NULL; + if (iree_status_is_ok(result)) { + result = iree_hal_inline_module_create( + instance, IREE_HAL_INLINE_MODULE_FLAG_NONE, + iree_hal_device_allocator(*device), host_allocator, &hal_inline_module); + } +#endif +#if defined(BUILD_INLINE_HAL) + iree_vm_module_t *modules[] = {hal_inline_module, module}; +#elif defined(BUILD_LOADER_HAL) + // Create hal_loader_module + iree_vm_module_t *hal_loader_module = NULL; + if (iree_status_is_ok(result)) { + result = iree_hal_loader_module_create(instance, IREE_HAL_MODULE_FLAG_NONE, + /*loader_count=*/1, &loader, + host_allocator, &hal_loader_module); + } + iree_hal_executable_loader_release(loader); + iree_vm_module_t *modules[] = {hal_inline_module, hal_loader_module, module}; +#else + // Create hal_module + iree_vm_module_t *hal_module = NULL; + if (iree_status_is_ok(result)) { + result = + iree_hal_module_create(instance, *device, IREE_HAL_MODULE_FLAG_NONE, + host_allocator, &hal_module); + } + iree_vm_module_t *modules[] = {hal_module, module}; +#endif + + // Allocate a context that will hold the module state across invocations. + if (iree_status_is_ok(result)) { + result = iree_vm_context_create_with_modules( + instance, IREE_VM_CONTEXT_FLAG_NONE, IREE_ARRAYSIZE(modules), + &modules[0], host_allocator, context); + } +#if defined(BUILD_INLINE_HAL) || defined(BUILD_LOADER_HAL) + iree_vm_module_release(hal_inline_module); +#else + iree_vm_module_release(hal_module); +#endif +#if defined(BUILD_LOADER_HAL) + iree_vm_module_release(hal_loader_module); +#endif + iree_vm_module_release(module); + return result; +} + // Prepare the input buffers and buffer_views based on the data type. They must // be released by the caller. static iree_status_t prepare_input_hal_buffer_views( @@ -76,36 +154,10 @@ iree_status_t run(const MlModel *model) { iree_vm_instance_t *instance = NULL; - iree_status_t result = - iree_vm_instance_create(iree_allocator_system(), &instance); - - IREE_RETURN_IF_ERROR(iree_hal_module_register_all_types(instance)); - iree_hal_device_t *device = NULL; - if (iree_status_is_ok(result)) { - result = create_sample_device(iree_allocator_system(), &device); - } - iree_vm_module_t *hal_module = NULL; - if (iree_status_is_ok(result)) { - result = iree_hal_module_create(instance, device, IREE_HAL_MODULE_FLAG_NONE, - iree_allocator_system(), &hal_module); - } - // Load bytecode or C module. - iree_vm_module_t *module = NULL; - if (iree_status_is_ok(result)) { - result = create_module(instance, &module); - } - - // Allocate a context that will hold the module state across invocations. iree_vm_context_t *context = NULL; - iree_vm_module_t *modules[] = {hal_module, module}; - if (iree_status_is_ok(result)) { - result = iree_vm_context_create_with_modules( - instance, IREE_VM_CONTEXT_FLAG_NONE, IREE_ARRAYSIZE(modules), - &modules[0], iree_allocator_system(), &context); - } - iree_vm_module_release(hal_module); - iree_vm_module_release(module); + // create context + iree_status_t result = create_context(instance, &device, &context); // Lookup the entry point function. // Note that we use the synchronous variant which operates on pure type/shape
diff --git a/samples/simple_vec_mul/CMakeLists.txt b/samples/simple_vec_mul/CMakeLists.txt index 9ea573e..e2eb95c 100644 --- a/samples/simple_vec_mul/CMakeLists.txt +++ b/samples/simple_vec_mul/CMakeLists.txt
@@ -16,6 +16,7 @@ "-riscv-v-vector-bits-min=512" "-riscv-v-fixed-length-vector-lmul-max=8" VMVX + INLINE_HAL PUBLIC ) @@ -31,6 +32,7 @@ "-riscv-v-vector-bits-min=512" "-riscv-v-fixed-length-vector-lmul-max=8" VMVX + INLINE_HAL PUBLIC ) @@ -38,6 +40,11 @@ # Binaries to execute the MLIR bytecode modules #------------------------------------------------------------------------------- +# Different util libraries available: +# util_static: static library using regular HAL +# util_vmvx: vmvx using regular HAL +# util_static_inline: static library using inline HAL +# util_vmvx_inline: vmvx using inline HAL # If the program requires a larger stack size, add # @@ -54,7 +61,7 @@ DEPS ::simple_float_mul_bytecode_module_vmvx_c iree::vm::bytecode_module - model_util::util_vmvx + model_util::util_vmvx_inline LINKOPTS "LINKER:--defsym=__stack_size__=20k" COPTS @@ -85,7 +92,7 @@ ::simple_float_mul_bytecode_module_static_c ::simple_float_mul_bytecode_module_static_lib iree::vm::bytecode_module - model_util::util_static + model_util::util_static_inline LINKOPTS "LINKER:--defsym=__stack_size__=20k" ) @@ -113,7 +120,7 @@ DEPS ::simple_int_mul_bytecode_module_vmvx_c iree::vm::bytecode_module - model_util::util_vmvx + model_util::util_vmvx_inline LINKOPTS "LINKER:--defsym=__stack_size__=20k" COPTS @@ -144,7 +151,7 @@ ::simple_int_mul_bytecode_module_static_c ::simple_int_mul_bytecode_module_static_lib iree::vm::bytecode_module - model_util::util_static + model_util::util_static_inline LINKOPTS "LINKER:--defsym=__stack_size__=20k" )