Adding a VM bytecode disassembler and flags for tracing to stderr. (#7261)
Tracing is enabled by default in debug builds or with
`-DIREE_VM_EXECUTION_TRACING_ENABLE=1` for others. A flag can then be
passed to iree_vm_context_t to enable tracing for all invocations
(including initializers) or passed per-invocation to trace only individual
calls. `--trace_execution` was added to iree-check-* and iree-run-*, but
all other tools/API usage can set
`-DIREE_VM_EXECUTION_TRACING_FORCE_ENABLE=1` to force it on.
Source locations can be printed per traced line too, however they are
only practically usable with vm source listings as otherwise they are
big multiline fused locs or call sites from python. Until some more
refinement of location printing can be done they are disabled but can be
turned back on with `-DIREE_VM_EXECUTION_TRACING_SRC_LOC_ENABLE=1`.
diff --git a/bindings/python/iree/runtime/vm.cc b/bindings/python/iree/runtime/vm.cc
index a5b15b0..a846d5e 100644
--- a/bindings/python/iree/runtime/vm.cc
+++ b/bindings/python/iree/runtime/vm.cc
@@ -78,8 +78,9 @@
iree_vm_context_t* context;
if (!modules) {
// Simple create with open allowed modules.
- auto status = iree_vm_context_create(instance->raw_ptr(),
- iree_allocator_system(), &context);
+ auto status =
+ iree_vm_context_create(instance->raw_ptr(), IREE_VM_CONTEXT_FLAG_NONE,
+ iree_allocator_system(), &context);
CheckApiStatus(status, "Error creating vm context");
} else {
// Closed set of modules.
@@ -89,8 +90,8 @@
module_handles[i] = (*modules)[i]->raw_ptr();
}
auto status = iree_vm_context_create_with_modules(
- instance->raw_ptr(), module_handles.data(), module_handles.size(),
- iree_allocator_system(), &context);
+ instance->raw_ptr(), IREE_VM_CONTEXT_FLAG_NONE, module_handles.data(),
+ module_handles.size(), iree_allocator_system(), &context);
CheckApiStatus(status, "Error creating vm context with modules");
}
@@ -111,8 +112,9 @@
void VmContext::Invoke(iree_vm_function_t f, VmVariantList& inputs,
VmVariantList& outputs) {
- CheckApiStatus(iree_vm_invoke(raw_ptr(), f, nullptr, inputs.raw_ptr(),
- outputs.raw_ptr(), iree_allocator_system()),
+ CheckApiStatus(iree_vm_invoke(raw_ptr(), f, IREE_VM_INVOCATION_FLAG_NONE,
+ nullptr, inputs.raw_ptr(), outputs.raw_ptr(),
+ iree_allocator_system()),
"Error invoking function");
}
diff --git a/bindings/tflite/interpreter.c b/bindings/tflite/interpreter.c
index 259293d..6ccf614 100644
--- a/bindings/tflite/interpreter.c
+++ b/bindings/tflite/interpreter.c
@@ -156,6 +156,7 @@
iree_vm_value_t index_value = iree_vm_value_make_i32(index);
IREE_IGNORE_ERROR(iree_vm_list_set_value(frame->arg_list, 0, &index_value));
return iree_vm_invoke(interpreter->context, apply_fn,
+ IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/NULL, frame->arg_list, /*outputs=*/NULL,
interpreter->allocator);
}
@@ -369,9 +370,9 @@
// tflite_resolver_module that we would register to resolve tflite ops into
// IREE functions that will call custom ops through TfLiteRegistrations.
IREE_RETURN_IF_ERROR(iree_vm_context_create_with_modules(
- interpreter->instance, interpreter->all_modules,
- IREE_ARRAYSIZE(interpreter->all_modules), interpreter->allocator,
- &interpreter->context));
+ interpreter->instance, IREE_VM_CONTEXT_FLAG_NONE,
+ interpreter->all_modules, IREE_ARRAYSIZE(interpreter->all_modules),
+ interpreter->allocator, &interpreter->context));
// Setup all I/O tensors and buffer views.
IREE_RETURN_IF_ERROR(_TfLiteInterpreterPopulateIO(interpreter));
@@ -444,6 +445,7 @@
interpreter->model->exports._reset_variables;
if (!iree_vm_function_is_null(reset_variables_fn)) {
status = iree_vm_invoke(interpreter->context, reset_variables_fn,
+ IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/NULL, /*inputs=*/NULL, /*outputs=*/NULL,
interpreter->allocator);
}
@@ -572,6 +574,7 @@
// emits it as '_main'.
IREE_RETURN_IF_ERROR(
iree_vm_invoke(interpreter->context, interpreter->model->exports._main,
+ IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/NULL, interpreter->input_list,
interpreter->output_list, interpreter->allocator));
diff --git a/docs/website/docs/bindings/c-api.md b/docs/website/docs/bindings/c-api.md
index 63e6ac1..174097d 100644
--- a/docs/website/docs/bindings/c-api.md
+++ b/docs/website/docs/bindings/c-api.md
@@ -162,8 +162,9 @@
iree_vm_context_t* context = NULL;
iree_vm_module_t* modules[2] = {hal_module, bytecode_module};
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- instance, modules, IREE_ARRAYSIZE(modules), iree_allocator_system(),
- &context));
+ instance, IREE_VM_CONTEXT_FLAG_NONE,
+ modules, IREE_ARRAYSIZE(modules),
+ iree_allocator_system(), &context));
// References to the modules can be released now.
iree_vm_module_release(hal_module);
iree_vm_module_release(bytecode_module);
diff --git a/iree/base/config.h b/iree/base/config.h
index f2be41c..f21dbd0 100644
--- a/iree/base/config.h
+++ b/iree/base/config.h
@@ -71,6 +71,7 @@
#if !defined(IREE_HOST_SIZE_T)
#define IREE_HOST_SIZE_T size_t
+#define PRIhsz "zu"
#endif // !IREE_HOST_SIZE_T
// Size, in bytes, of a buffer on the local host.
@@ -178,6 +179,36 @@
#define IREE_VM_BACKTRACE_ENABLE 1
#endif // !IREE_VM_BACKTRACE_ENABLE
+#if !defined(IREE_VM_EXECUTION_TRACING_ENABLE)
+// Enables disassembly of vm bytecode functions and stderr dumping of execution.
+// Increases code size quite, lowers VM performance, and is generally unsafe;
+// include only when debugging or running on trusted inputs.
+#ifdef NDEBUG
+#define IREE_VM_EXECUTION_TRACING_ENABLE 0
+#else
+#define IREE_VM_EXECUTION_TRACING_ENABLE 1
+#endif // NDEBUG
+#endif // !IREE_VM_EXECUTION_TRACING_ENABLE
+
+#if !defined(IREE_VM_EXECUTION_TRACING_FORCE_ENABLE)
+// Forces tracing of VM execution by default ignoring runtime flags that may
+// otherwise control the behavior. This can be used to enable tracing in tools
+// that do not have flag parsing or plumbing for per-invocation flags.
+#define IREE_VM_EXECUTION_TRACING_FORCE_ENABLE 0
+#endif // !IREE_VM_EXECUTION_TRACING_FORCE_ENABLE
+#if defined(IREE_VM_EXECUTION_TRACING_FORCE_ENABLE)
+#define IREE_VM_EXECUTION_TRACING_ENABLE 1
+#endif // IREE_VM_EXECUTION_TRACING_FORCE_ENABLE
+
+#if !defined(IREE_VM_EXECUTION_TRACING_SRC_LOC_ENABLE)
+// Enables printing of the source location of an op when tracing its execution.
+// This may be messy depending on the origin of the locations in the program;
+// for example today the python locs are entire stack traces. Improvements to
+// printing of more complex source locations (or a way to prune them in the
+// compiler) would let this be turned on by default.
+#define IREE_VM_EXECUTION_TRACING_SRC_LOC_ENABLE 0
+#endif // !IREE_VM_EXECUTION_TRACING_SRC_LOC_ENABLE
+
#if !defined(IREE_VM_EXT_I64_ENABLE)
// Enables the 64-bit integer instruction extension.
// Targeted from the compiler with `-iree-vm-target-extension-i64`.
diff --git a/iree/hal/local/loaders/vmvx_module_loader.c b/iree/hal/local/loaders/vmvx_module_loader.c
index 036daa2..4b3060e 100644
--- a/iree/hal/local/loaders/vmvx_module_loader.c
+++ b/iree/hal/local/loaders/vmvx_module_loader.c
@@ -300,7 +300,8 @@
// On-stack stack. We really do abuse the stack too much here.
// TODO(benvanik): pass in an iree_arena_t that can be used for this.
IREE_VM_INLINE_STACK_INITIALIZE(
- stack, iree_vm_context_state_resolver(executable->context),
+ stack, IREE_VM_INVOCATION_FLAG_NONE,
+ iree_vm_context_state_resolver(executable->context),
executable->base.host_allocator);
// Direct call interface.
@@ -448,8 +449,8 @@
bytecode_module,
};
status = iree_vm_context_create_with_modules(
- executable_loader->instance, modules, IREE_ARRAYSIZE(modules),
- executable_loader->host_allocator, &context);
+ executable_loader->instance, IREE_VM_CONTEXT_FLAG_NONE, modules,
+ IREE_ARRAYSIZE(modules), executable_loader->host_allocator, &context);
}
// Executable takes ownership of the entire context (including the bytecode
diff --git a/iree/modules/check/check_test.cc b/iree/modules/check/check_test.cc
index dbf03c0..404f016 100644
--- a/iree/modules/check/check_test.cc
+++ b/iree/modules/check/check_test.cc
@@ -62,8 +62,8 @@
void SetUp() override {
std::vector<iree_vm_module_t*> modules = {hal_module_, check_module_};
IREE_ASSERT_OK(iree_vm_context_create_with_modules(
- instance_, modules.data(), modules.size(), iree_allocator_system(),
- &context_));
+ instance_, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &context_));
allocator_ = iree_hal_device_allocator(device_);
}
@@ -172,7 +172,7 @@
iree_make_cstring_view(function_name), &function),
"exported function '%s' not found", function_name);
// TODO(#2075): don't directly invoke native functions like this.
- return iree_vm_invoke(context_, function,
+ return iree_vm_invoke(context_, function, IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/nullptr, inputs_.get(),
/*outputs=*/nullptr, iree_allocator_system());
}
diff --git a/iree/runtime/session.c b/iree/runtime/session.c
index ba40aa4..8704654 100644
--- a/iree/runtime/session.c
+++ b/iree/runtime/session.c
@@ -25,6 +25,7 @@
IREE_API_EXPORT void iree_runtime_session_options_initialize(
iree_runtime_session_options_t* out_options) {
memset(out_options, 0, sizeof(*out_options));
+ out_options->context_flags = IREE_VM_CONTEXT_FLAG_NONE;
out_options->builtin_modules = IREE_RUNTIME_SESSION_BUILTIN_ALL;
}
@@ -86,7 +87,8 @@
// Create the context empty so that we can add our modules to it.
iree_status_t status = iree_vm_context_create(
- /*instance=*/NULL, host_allocator, &session->context);
+ /*instance=*/NULL, options->context_flags, host_allocator,
+ &session->context);
// Add the HAL module; it is always required when using the runtime API.
// Lower-level usage of the VM can avoid the HAL if it's not required.
@@ -257,6 +259,7 @@
iree_status_t status =
iree_vm_invoke(iree_runtime_session_context(session), *function,
+ IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/NULL, input_list, output_list,
iree_runtime_session_host_allocator(session));
@@ -282,7 +285,7 @@
// Allocate a VM stack on the host stack and initialize it.
IREE_VM_INLINE_STACK_INITIALIZE(
- stack,
+ stack, IREE_VM_INVOCATION_FLAG_NONE,
iree_vm_context_state_resolver(iree_runtime_session_context(session)),
iree_runtime_session_host_allocator(session));
diff --git a/iree/runtime/session.h b/iree/runtime/session.h
index 0172b58..960d7fb 100644
--- a/iree/runtime/session.h
+++ b/iree/runtime/session.h
@@ -54,6 +54,9 @@
// Options used to configure session creation.
typedef struct iree_runtime_session_options_t {
+ // Flags controlling the execution environment.
+ iree_vm_context_flags_t context_flags;
+
// A bitmask identifying which IREE builtin modules should be enabled.
// Session creation will fail if a requested module is not built into the
// runtime binary.
diff --git a/iree/samples/custom_modules/custom_modules_test.cc b/iree/samples/custom_modules/custom_modules_test.cc
index 10f9b15..7e37166 100644
--- a/iree/samples/custom_modules/custom_modules_test.cc
+++ b/iree/samples/custom_modules/custom_modules_test.cc
@@ -61,8 +61,8 @@
std::vector<iree_vm_module_t*> modules = {hal_module_, native_module_,
bytecode_module_};
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- instance_, modules.data(), modules.size(), iree_allocator_system(),
- &context_));
+ instance_, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &context_));
}
virtual void TearDown() {
@@ -113,6 +113,7 @@
// Synchronously invoke the function.
IREE_ASSERT_OK(iree_vm_invoke(context_, LookupFunction("reverseAndPrint"),
+ IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/nullptr, inputs.get(), outputs.get(),
iree_allocator_system()));
@@ -157,6 +158,7 @@
// Synchronously invoke the function.
IREE_ASSERT_OK(iree_vm_invoke(context_, LookupFunction("printTensor"),
+ IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/nullptr, inputs.get(), outputs.get(),
iree_allocator_system()));
@@ -201,6 +203,7 @@
// Synchronously invoke the function.
IREE_ASSERT_OK(iree_vm_invoke(context_, LookupFunction("roundTripTensor"),
+ IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/nullptr, inputs.get(), outputs.get(),
iree_allocator_system()));
diff --git a/iree/samples/emitc_modules/add_module_test.cc b/iree/samples/emitc_modules/add_module_test.cc
index 12712c1..3b28999 100644
--- a/iree/samples/emitc_modules/add_module_test.cc
+++ b/iree/samples/emitc_modules/add_module_test.cc
@@ -25,8 +25,8 @@
std::vector<iree_vm_module_t*> modules = {add_module};
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- instance_, modules.data(), modules.size(), iree_allocator_system(),
- &context_));
+ instance_, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &context_));
iree_vm_module_release(add_module);
}
@@ -56,10 +56,10 @@
/*element_type=*/nullptr, 1, iree_allocator_system(), &output_list));
// Invoke the entry function to do our work. Runs synchronously.
- IREE_RETURN_IF_ERROR(iree_vm_invoke(context_, function,
- /*policy=*/nullptr, input_list.get(),
- output_list.get(),
- iree_allocator_system()));
+ IREE_RETURN_IF_ERROR(
+ iree_vm_invoke(context_, function, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/nullptr, input_list.get(), output_list.get(),
+ iree_allocator_system()));
// Load the output result.
iree_vm_value_t ret_value;
@@ -93,10 +93,10 @@
/*element_type=*/nullptr, 1, iree_allocator_system(), &output_list));
// Invoke the entry function to do our work. Runs synchronously.
- IREE_RETURN_IF_ERROR(iree_vm_invoke(context_, function,
- /*policy=*/nullptr, input_list.get(),
- output_list.get(),
- iree_allocator_system()));
+ IREE_RETURN_IF_ERROR(
+ iree_vm_invoke(context_, function, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/nullptr, input_list.get(), output_list.get(),
+ iree_allocator_system()));
// Load the output result.
iree_vm_value_t ret_value;
diff --git a/iree/samples/emitc_modules/import_module_test.cc b/iree/samples/emitc_modules/import_module_test.cc
index 86ec36f..b8218b6 100644
--- a/iree/samples/emitc_modules/import_module_test.cc
+++ b/iree/samples/emitc_modules/import_module_test.cc
@@ -29,8 +29,8 @@
// Note: order matters as module_a imports from module_b
std::vector<iree_vm_module_t*> modules = {module_b, module_a};
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- instance_, modules.data(), modules.size(), iree_allocator_system(),
- &context_));
+ instance_, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &context_));
iree_vm_module_release(module_a);
iree_vm_module_release(module_b);
@@ -61,10 +61,10 @@
/*element_type=*/nullptr, 1, iree_allocator_system(), &output_list));
// Invoke the entry function to do our work. Runs synchronously.
- IREE_RETURN_IF_ERROR(iree_vm_invoke(context_, function,
- /*policy=*/nullptr, input_list.get(),
- output_list.get(),
- iree_allocator_system()));
+ IREE_RETURN_IF_ERROR(
+ iree_vm_invoke(context_, function, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/nullptr, input_list.get(), output_list.get(),
+ iree_allocator_system()));
// Load the output result.
iree_vm_value_t ret_value;
diff --git a/iree/samples/simple_embedding/simple_embedding.c b/iree/samples/simple_embedding/simple_embedding.c
index c04a15e..b9ce44d 100644
--- a/iree/samples/simple_embedding/simple_embedding.c
+++ b/iree/samples/simple_embedding/simple_embedding.c
@@ -51,8 +51,8 @@
iree_vm_context_t* context = NULL;
iree_vm_module_t* modules[] = {hal_module, bytecode_module};
IREE_RETURN_IF_ERROR(iree_vm_context_create_with_modules(
- instance, &modules[0], IREE_ARRAYSIZE(modules), iree_allocator_system(),
- &context));
+ instance, IREE_VM_CONTEXT_FLAG_NONE, &modules[0], IREE_ARRAYSIZE(modules),
+ iree_allocator_system(), &context));
iree_vm_module_release(hal_module);
iree_vm_module_release(bytecode_module);
@@ -124,9 +124,9 @@
"can't allocate output vm list");
// Synchronously invoke the function.
- IREE_RETURN_IF_ERROR(iree_vm_invoke(context, main_function,
- /*policy=*/NULL, inputs, outputs,
- iree_allocator_system()));
+ IREE_RETURN_IF_ERROR(iree_vm_invoke(
+ context, main_function, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/NULL, inputs, outputs, iree_allocator_system()));
// Get the result buffers from the invocation.
iree_hal_buffer_view_t* ret_buffer_view =
diff --git a/iree/samples/vulkan/vulkan_inference_gui.cc b/iree/samples/vulkan/vulkan_inference_gui.cc
index f520a35..70124ed 100644
--- a/iree/samples/vulkan/vulkan_inference_gui.cc
+++ b/iree/samples/vulkan/vulkan_inference_gui.cc
@@ -258,8 +258,8 @@
iree_vm_context_t* iree_context = nullptr;
std::vector<iree_vm_module_t*> modules = {hal_module, bytecode_module};
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- iree_instance, modules.data(), modules.size(), iree_allocator_system(),
- &iree_context));
+ iree_instance, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &iree_context));
IREE_LOG(INFO) << "Context with modules is ready for use";
// Lookup the async entry point function.
@@ -431,6 +431,7 @@
// Asynchronously invoke the function.
IREE_CHECK_OK(iree_vm_invoke(iree_context, main_function,
+ IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/nullptr, inputs.get(),
outputs.get(), iree_allocator_system()));
diff --git a/iree/testing/vulkan/iree-run-module-vulkan-gui-main.cc b/iree/testing/vulkan/iree-run-module-vulkan-gui-main.cc
index d042ac6..961efad 100644
--- a/iree/testing/vulkan/iree-run-module-vulkan-gui-main.cc
+++ b/iree/testing/vulkan/iree-run-module-vulkan-gui-main.cc
@@ -117,9 +117,9 @@
iree_allocator_system(), &outputs));
IREE_LOG(INFO) << "EXEC @" << function_name;
- IREE_RETURN_IF_ERROR(iree_vm_invoke(context, function, /*policy=*/nullptr,
- function_inputs.get(), outputs.get(),
- iree_allocator_system()));
+ IREE_RETURN_IF_ERROR(iree_vm_invoke(
+ context, function, IREE_VM_INVOCATION_FLAG_NONE, /*policy=*/nullptr,
+ function_inputs.get(), outputs.get(), iree_allocator_system()));
std::ostringstream oss;
IREE_RETURN_IF_ERROR(PrintVariantList(outputs.get(), &oss));
@@ -320,8 +320,8 @@
iree_vm_context_t* iree_context = nullptr;
std::vector<iree_vm_module_t*> modules = {hal_module, bytecode_module};
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- iree_instance, modules.data(), modules.size(), iree_allocator_system(),
- &iree_context));
+ iree_instance, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &iree_context));
IREE_LOG(INFO) << "Context with modules is ready for use";
// Lookup the entry point function.
diff --git a/iree/tools/android/run_module_app/src/main.cc b/iree/tools/android/run_module_app/src/main.cc
index a29cf15..fb16c42 100644
--- a/iree/tools/android/run_module_app/src/main.cc
+++ b/iree/tools/android/run_module_app/src/main.cc
@@ -111,8 +111,8 @@
// Order matters. The input module will likely be dependent on the hal module.
std::array<iree_vm_module_t*, 2> modules = {hal_module, input_module};
IREE_RETURN_IF_ERROR(iree_vm_context_create_with_modules(
- instance, modules.data(), modules.size(),
- iree_allocator_system(), &context),
+ instance, IREE_VM_CONTEXT_FLAG_NONE, modules.data(),
+ modules.size(), iree_allocator_system(), &context),
"creating context");
const std::string& function_name = invocation.entry_function;
@@ -141,8 +141,9 @@
LOGI("Execute @%s", function_name.c_str());
IREE_RETURN_IF_ERROR(
- iree_vm_invoke(context, function, /*policy=*/nullptr, inputs.get(),
- outputs.get(), iree_allocator_system()),
+ iree_vm_invoke(context, function, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/nullptr, inputs.get(), outputs.get(),
+ iree_allocator_system()),
"invoking function '%s'", function_name.c_str());
std::ostringstream oss;
diff --git a/iree/tools/iree-benchmark-module-main.cc b/iree/tools/iree-benchmark-module-main.cc
index 04c7e65..1ac9432 100644
--- a/iree/tools/iree-benchmark-module-main.cc
+++ b/iree/tools/iree-benchmark-module-main.cc
@@ -96,8 +96,9 @@
vm::ref<iree_vm_list_t> outputs;
IREE_CHECK_OK(iree_vm_list_create(/*element_type=*/nullptr, 16,
iree_allocator_system(), &outputs));
- IREE_CHECK_OK(iree_vm_invoke(context, function, /*policy=*/nullptr, inputs,
- outputs.get(), iree_allocator_system()));
+ IREE_CHECK_OK(iree_vm_invoke(
+ context, function, IREE_VM_INVOCATION_FLAG_NONE, /*policy=*/nullptr,
+ inputs, outputs.get(), iree_allocator_system()));
}
}
@@ -207,8 +208,8 @@
// module.
std::array<iree_vm_module_t*, 2> modules = {hal_module_, input_module_};
IREE_RETURN_IF_ERROR(iree_vm_context_create_with_modules(
- instance_, modules.data(), modules.size(), iree_allocator_system(),
- &context_));
+ instance_, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &context_));
IREE_TRACE_FRAME_MARK_END_NAMED("init");
return iree_ok_status();
diff --git a/iree/tools/iree-benchmark-trace-main.c b/iree/tools/iree-benchmark-trace-main.c
index 7381eec..d4ee89e 100644
--- a/iree/tools/iree-benchmark-trace-main.c
+++ b/iree/tools/iree-benchmark-trace-main.c
@@ -197,8 +197,8 @@
// Setup replay state used for this benchmark.
iree_trace_replay_t replay;
IREE_RETURN_IF_ERROR(iree_trace_replay_initialize(
- registration->root_path, registration->instance, iree_allocator_system(),
- &replay));
+ registration->root_path, registration->instance,
+ IREE_VM_CONTEXT_FLAG_NONE, iree_allocator_system(), &replay));
iree_trace_replay_set_hal_driver_override(
&replay, iree_make_cstring_view(FLAG_driver));
@@ -214,10 +214,10 @@
for (size_t i = 0; i < call_list.count; ++i) {
iree_replay_benchmark_call_t* call = &call_list.items[i];
for (int32_t j = 0; j < FLAG_call_iterations; ++j) {
- IREE_RETURN_IF_ERROR(iree_vm_invoke(replay.context, call->function,
- /*policy=*/NULL, call->input_list,
- call->output_list,
- replay.host_allocator));
+ IREE_RETURN_IF_ERROR(iree_vm_invoke(
+ replay.context, call->function, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/NULL, call->input_list, call->output_list,
+ replay.host_allocator));
IREE_RETURN_IF_ERROR(iree_vm_list_resize(call->output_list, 0));
}
}
diff --git a/iree/tools/iree-check-module-main.cc b/iree/tools/iree-check-module-main.cc
index a8f14e6..a0bf2e3 100644
--- a/iree/tools/iree-check-module-main.cc
+++ b/iree/tools/iree-check-module-main.cc
@@ -40,6 +40,8 @@
#define IREE_FORCE_BINARY_STDIN()
#endif // IREE_PLATFORM_WINDOWS
+IREE_FLAG(bool, trace_execution, false, "Traces VM execution to stderr.");
+
IREE_FLAG(string, driver, "vmvx", "Backend driver to use.");
IREE_FLAG(
@@ -60,15 +62,18 @@
: instance_(instance), modules_(modules), function_(function) {}
void SetUp() override {
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- instance_, modules_.data(), modules_.size(), iree_allocator_system(),
- &context_));
+ instance_,
+ FLAG_trace_execution ? IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION
+ : IREE_VM_CONTEXT_FLAG_NONE,
+ modules_.data(), modules_.size(), iree_allocator_system(), &context_));
}
void TearDown() override { iree_vm_context_release(context_); }
void TestBody() override {
- IREE_EXPECT_OK(iree_vm_invoke(context_, function_, /*policy=*/nullptr,
- /*inputs=*/nullptr, /*outputs=*/nullptr,
- iree_allocator_system()));
+ IREE_EXPECT_OK(iree_vm_invoke(
+ context_, function_, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/nullptr,
+ /*inputs=*/nullptr, /*outputs=*/nullptr, iree_allocator_system()));
}
private:
diff --git a/iree/tools/iree-run-mlir-main.cc b/iree/tools/iree-run-mlir-main.cc
index cd695dd..e3a62a6 100644
--- a/iree/tools/iree-run-mlir-main.cc
+++ b/iree/tools/iree-run-mlir-main.cc
@@ -98,7 +98,7 @@
llvm::cl::init(true),
};
-static llvm::cl::opt<bool> verifyPasses(
+static llvm::cl::opt<bool> verify_passes_flag(
"verify-each",
llvm::cl::desc("Run the verifier after each transformation pass"),
llvm::cl::init(true));
@@ -139,6 +139,12 @@
llvm::cl::ZeroOrMore,
};
+static llvm::cl::opt<bool> trace_execution_flag{
+ "trace-execution",
+ llvm::cl::desc("Traces VM execution to stderr"),
+ llvm::cl::init(false),
+};
+
namespace iree {
namespace {
@@ -200,7 +206,7 @@
IREE_LOG(INFO) << "Compiling for target backend '" << target_backend
<< "'...";
mlir::PassManager pass_manager(mlir_module->getContext());
- pass_manager.enableVerifier(verifyPasses);
+ pass_manager.enableVerifier(verify_passes_flag);
mlir::applyPassManagerCLOptions(pass_manager);
mlir::applyDefaultTimingPassManagerCLOptions(pass_manager);
mlir::iree_compiler::buildDefaultIREEVMTransformPassPipeline(pass_manager);
@@ -283,9 +289,10 @@
iree_allocator_system(), &outputs));
// Synchronously invoke the function.
- IREE_RETURN_IF_ERROR(iree_vm_invoke(context, function, /*policy=*/nullptr,
- inputs.get(), outputs.get(),
- iree_allocator_system()));
+ IREE_RETURN_IF_ERROR(iree_vm_invoke(context, function,
+ IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/nullptr, inputs.get(),
+ outputs.get(), iree_allocator_system()));
// Print outputs.
IREE_RETURN_IF_ERROR(PrintVariantList(outputs.get()));
@@ -344,10 +351,13 @@
// runner).
iree_vm_context_t* context = nullptr;
std::vector<iree_vm_module_t*> modules = {hal_module, bytecode_module};
- IREE_RETURN_IF_ERROR(iree_vm_context_create_with_modules(
- instance, modules.data(), modules.size(),
- iree_allocator_system(), &context),
- "Creating context");
+ IREE_RETURN_IF_ERROR(
+ iree_vm_context_create_with_modules(
+ instance,
+ trace_execution_flag ? IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION
+ : IREE_VM_CONTEXT_FLAG_NONE,
+ modules.data(), modules.size(), iree_allocator_system(), &context),
+ "Creating context");
// Invoke the function and print results.
IREE_RETURN_IF_ERROR(
diff --git a/iree/tools/iree-run-module-main.cc b/iree/tools/iree-run-module-main.cc
index 1604219..ec24203 100644
--- a/iree/tools/iree-run-module-main.cc
+++ b/iree/tools/iree-run-module-main.cc
@@ -33,6 +33,8 @@
"Name of a function contained in the module specified by module_file "
"to run.");
+IREE_FLAG(bool, trace_execution, false, "Traces VM execution to stderr.");
+
IREE_FLAG(string, driver, "vmvx", "Backend driver to use.");
IREE_FLAG(bool, print_statistics, false,
@@ -110,10 +112,13 @@
iree_vm_context_t* context = nullptr;
// Order matters. The input module will likely be dependent on the hal module.
std::array<iree_vm_module_t*, 2> modules = {hal_module, input_module};
- IREE_RETURN_IF_ERROR(iree_vm_context_create_with_modules(
- instance, modules.data(), modules.size(),
- iree_allocator_system(), &context),
- "creating context");
+ IREE_RETURN_IF_ERROR(
+ iree_vm_context_create_with_modules(
+ instance,
+ FLAG_trace_execution ? IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION
+ : IREE_VM_CONTEXT_FLAG_NONE,
+ modules.data(), modules.size(), iree_allocator_system(), &context),
+ "creating context");
std::string function_name = std::string(FLAG_entry_function);
iree_vm_function_t function;
@@ -142,8 +147,9 @@
std::cout << "EXEC @" << function_name << "\n";
IREE_RETURN_IF_ERROR(
- iree_vm_invoke(context, function, /*policy=*/nullptr, inputs.get(),
- outputs.get(), iree_allocator_system()),
+ iree_vm_invoke(context, function, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/nullptr, inputs.get(), outputs.get(),
+ iree_allocator_system()),
"invoking function '%s'", function_name.c_str());
IREE_RETURN_IF_ERROR(PrintVariantList(outputs.get()), "printing results");
diff --git a/iree/tools/iree-run-trace-main.c b/iree/tools/iree-run-trace-main.c
index b8bb538..d27e799 100644
--- a/iree/tools/iree-run-trace-main.c
+++ b/iree/tools/iree-run-trace-main.c
@@ -17,6 +17,8 @@
#include "iree/tools/utils/yaml_util.h"
#include "iree/vm/api.h"
+IREE_FLAG(bool, trace_execution, false, "Traces VM execution to stderr.");
+
IREE_FLAG(string, driver, "vmvx", "Backend driver to use.");
// Runs the trace in |file| using |root_path| as the base for any path lookups
@@ -26,7 +28,10 @@
iree_vm_instance_t* instance) {
iree_trace_replay_t replay;
IREE_RETURN_IF_ERROR(iree_trace_replay_initialize(
- root_path, instance, iree_allocator_system(), &replay));
+ root_path, instance,
+ FLAG_trace_execution ? IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION
+ : IREE_VM_CONTEXT_FLAG_NONE,
+ iree_allocator_system(), &replay));
iree_trace_replay_set_hal_driver_override(
&replay, iree_make_cstring_view(FLAG_driver));
diff --git a/iree/tools/utils/trace_replay.c b/iree/tools/utils/trace_replay.c
index cba6e5f..9e91d62 100644
--- a/iree/tools/utils/trace_replay.c
+++ b/iree/tools/utils/trace_replay.c
@@ -18,16 +18,17 @@
#include "iree/modules/hal/module.h"
#include "iree/vm/bytecode_module.h"
-iree_status_t iree_trace_replay_initialize(iree_string_view_t root_path,
- iree_vm_instance_t* instance,
- iree_allocator_t host_allocator,
- iree_trace_replay_t* out_replay) {
+iree_status_t iree_trace_replay_initialize(
+ iree_string_view_t root_path, iree_vm_instance_t* instance,
+ iree_vm_context_flags_t context_flags, iree_allocator_t host_allocator,
+ iree_trace_replay_t* out_replay) {
memset(out_replay, 0, sizeof(*out_replay));
IREE_RETURN_IF_ERROR(iree_hal_module_register_types());
out_replay->root_path = root_path;
out_replay->instance = instance;
+ out_replay->context_flags = context_flags;
out_replay->host_allocator = host_allocator;
iree_vm_instance_retain(out_replay->instance);
return iree_ok_status();
@@ -55,8 +56,9 @@
replay->context = NULL;
// Create new context.
- return iree_vm_context_create(replay->instance, replay->host_allocator,
- &replay->context);
+ // TODO(benvanik): allow setting flags from the trace files.
+ return iree_vm_context_create(replay->instance, replay->context_flags,
+ replay->host_allocator, &replay->context);
}
static iree_status_t iree_trace_replay_create_device(
@@ -765,7 +767,8 @@
iree_vm_list_create(/*element_type=*/NULL, /*initial_capacity=*/8,
replay->host_allocator, &output_list);
if (iree_status_is_ok(status)) {
- status = iree_vm_invoke(replay->context, function, /*policy=*/NULL,
+ status = iree_vm_invoke(replay->context, function,
+ IREE_VM_INVOCATION_FLAG_NONE, /*policy=*/NULL,
input_list, output_list, replay->host_allocator);
}
iree_vm_list_release(input_list);
@@ -801,7 +804,8 @@
iree_vm_list_create(/*element_type=*/NULL, /*initial_capacity=*/8,
replay->host_allocator, &output_list);
if (iree_status_is_ok(status)) {
- status = iree_vm_invoke(replay->context, function, /*policy=*/NULL,
+ status = iree_vm_invoke(replay->context, function,
+ IREE_VM_INVOCATION_FLAG_NONE, /*policy=*/NULL,
input_list, output_list, replay->host_allocator);
}
iree_vm_list_release(input_list);
diff --git a/iree/tools/utils/trace_replay.h b/iree/tools/utils/trace_replay.h
index 356e327..b1a1fb8 100644
--- a/iree/tools/utils/trace_replay.h
+++ b/iree/tools/utils/trace_replay.h
@@ -20,6 +20,7 @@
iree_allocator_t host_allocator;
iree_string_view_t root_path;
iree_vm_instance_t* instance;
+ iree_vm_context_flags_t context_flags;
iree_string_view_t driver;
iree_vm_context_t* context;
@@ -29,10 +30,10 @@
// Initializes a trace replay context.
// Relative paths will be joined with |root_path| to form a fully-qualified
// path (may be cwd, may be file source, etc).
-iree_status_t iree_trace_replay_initialize(iree_string_view_t root_path,
- iree_vm_instance_t* instance,
- iree_allocator_t host_allocator,
- iree_trace_replay_t* out_replay);
+iree_status_t iree_trace_replay_initialize(
+ iree_string_view_t root_path, iree_vm_instance_t* instance,
+ iree_vm_context_flags_t context_flags, iree_allocator_t host_allocator,
+ iree_trace_replay_t* out_replay);
// Deinitializes a trace replay context and releases all resources.
void iree_trace_replay_deinitialize(iree_trace_replay_t* replay);
diff --git a/iree/vm/BUILD b/iree/vm/BUILD
index daac376..bc5a4df 100644
--- a/iree/vm/BUILD
+++ b/iree/vm/BUILD
@@ -181,6 +181,8 @@
cc_library(
name = "bytecode_module",
srcs = [
+ "bytecode_disasm.c",
+ "bytecode_disasm.h",
"bytecode_dispatch.c",
"bytecode_dispatch_util.h",
"bytecode_module.c",
diff --git a/iree/vm/CMakeLists.txt b/iree/vm/CMakeLists.txt
index 0a239e4..2d34cdf 100644
--- a/iree/vm/CMakeLists.txt
+++ b/iree/vm/CMakeLists.txt
@@ -171,6 +171,8 @@
HDRS
"bytecode_module.h"
SRCS
+ "bytecode_disasm.c"
+ "bytecode_disasm.h"
"bytecode_dispatch.c"
"bytecode_dispatch_util.h"
"bytecode_module.c"
diff --git a/iree/vm/bytecode_disasm.c b/iree/vm/bytecode_disasm.c
new file mode 100644
index 0000000..1bdff07
--- /dev/null
+++ b/iree/vm/bytecode_disasm.c
@@ -0,0 +1,2204 @@
+// Copyright 2021 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#include "iree/vm/bytecode_disasm.h"
+
+#include <inttypes.h>
+
+#include "iree/base/config.h"
+#include "iree/vm/ops.h"
+
+#define BEGIN_DISASM_PREFIX(op_name, ext) \
+ case IREE_VM_OP_CORE_##op_name: { \
+ switch (bytecode_data[pc++]) {
+#define END_DISASM_PREFIX() \
+ default: \
+ return iree_make_status(IREE_STATUS_UNIMPLEMENTED, \
+ "unhandled ext opcode"); \
+ } \
+ break; \
+ }
+#define UNHANDLED_DISASM_PREFIX(op_name, ext) \
+ case IREE_VM_OP_CORE_##op_name: { \
+ return iree_make_status(IREE_STATUS_UNIMPLEMENTED, \
+ "unhandled dispatch extension " #ext); \
+ }
+
+#define DISASM_OP(ext, op_name) case IREE_VM_OP_##ext##_##op_name:
+
+#define VM_ParseConstI8(name) \
+ OP_I8(0); \
+ ++pc;
+#define VM_ParseConstI32(name) \
+ OP_I32(0); \
+ pc += 4;
+#define VM_ParseConstI64(name) \
+ OP_I64(0); \
+ pc += 8;
+#define VM_ParseConstF32(name) \
+ OP_F32(0); \
+ pc += 4;
+#define VM_ParseConstF64(name) \
+ OP_F64(0); \
+ pc += 8;
+#define VM_ParseOpcode(opcode) VM_ParseConstI8(#opcode)
+#define VM_ParseFuncAttr(name) VM_ParseConstI32(name)
+#define VM_ParseGlobalAttr(name) VM_ParseConstI32(name)
+#define VM_ParseRodataAttr(name) VM_ParseConstI32(name)
+#define VM_ParseType(name) \
+ iree_vm_map_type(module, OP_I32(0)); \
+ pc += 4;
+#define VM_ParseTypeOf(name) VM_ParseType(name)
+#define VM_ParseIntAttr32(name) VM_ParseConstI32(name)
+#define VM_ParseIntAttr64(name) VM_ParseConstI64(name)
+#define VM_ParseFloatAttr32(name) VM_ParseConstF32(name)
+#define VM_ParseFloatAttr64(name) VM_ParseConstF64(name)
+#define VM_ParseStrAttr(name, out_str) \
+ (out_str)->size = (iree_host_size_t)OP_I16(0); \
+ (out_str)->data = (const char*)&bytecode_data[pc + 2]; \
+ pc += 2 + (out_str)->size;
+#define VM_ParseBranchTarget(block_name) VM_ParseConstI32(name)
+#define VM_ParseBranchOperands(operands_name) \
+ VM_DecBranchOperandsImpl(bytecode_data, &pc)
+#define VM_ParseOperandRegI32(name) \
+ OP_I16(0) & regs->i32_mask; \
+ pc += kRegSize;
+#define VM_ParseOperandRegI64(name) \
+ OP_I16(0) & (regs->i32_mask & ~1); \
+ pc += kRegSize;
+#define VM_ParseOperandRegF32(name) \
+ OP_I16(0) & regs->i32_mask; \
+ pc += kRegSize;
+#define VM_ParseOperandRegF64(name) \
+ OP_I16(0) & (regs->i32_mask & ~1); \
+ pc += kRegSize;
+#define VM_ParseOperandRegRef(name, out_is_move) \
+ OP_I16(0) & regs->ref_mask; \
+ *(out_is_move) = 0; /*= OP_I16(0) & IREE_REF_REGISTER_MOVE_BIT;*/ \
+ pc += kRegSize;
+#define VM_ParseVariadicOperands(name) \
+ VM_DecVariadicOperandsImpl(bytecode_data, &pc)
+#define VM_ParseResultRegI32(name) \
+ OP_I16(0) & regs->i32_mask; \
+ pc += kRegSize;
+#define VM_ParseResultRegI64(name) \
+ OP_I16(0) & (regs->i32_mask & ~1); \
+ pc += kRegSize;
+#define VM_ParseResultRegF32(name) \
+ OP_I16(0) & regs->i32_mask; \
+ pc += kRegSize;
+#define VM_ParseResultRegF64(name) \
+ OP_I16(0) & (regs->i32_mask & ~1); \
+ pc += kRegSize;
+#define VM_ParseResultRegRef(name, out_is_move) \
+ OP_I16(0) & regs->ref_mask; \
+ *(out_is_move) = 0; /*= OP_I16(0) & IREE_REF_REGISTER_MOVE_BIT;*/ \
+ pc += kRegSize;
+#define VM_ParseVariadicResults(name) VM_ParseVariadicOperands(name)
+
+#define EMIT_REG_NAME(reg) \
+ if ((reg)&IREE_REF_REGISTER_TYPE_BIT) { \
+ EMIT_REF_REG_NAME(reg); \
+ } else { \
+ EMIT_I32_REG_NAME(reg); \
+ }
+#define EMIT_I32_REG_NAME(reg) \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format( \
+ b, "%%i%u", ((reg)&IREE_I32_REGISTER_MASK)));
+#define EMIT_I64_REG_NAME(reg) \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format( \
+ b, "%%i%u:%u", ((reg)&IREE_I32_REGISTER_MASK), \
+ ((reg)&IREE_I32_REGISTER_MASK) + 1));
+#define EMIT_F32_REG_NAME(reg) EMIT_I32_REG_NAME(reg)
+#define EMIT_REF_REG_NAME(reg) \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format( \
+ b, "%%r%u", ((reg)&IREE_REF_REGISTER_MASK)));
+
+#define EMIT_REG_VALUE(regs, reg) \
+ if ((reg)&IREE_REF_REGISTER_TYPE_BIT) { \
+ iree_vm_ref_t* ref = &(regs)->ref[(reg)&IREE_REF_REGISTER_MASK]; \
+ if (iree_vm_ref_is_null(ref)) { \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "null")); \
+ } else { \
+ iree_string_view_t type_name = iree_vm_ref_type_name(ref->type); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format( \
+ b, "!%.*s/0x%p", (int)type_name.size, type_name.data, ref->ptr)); \
+ } \
+ } else { \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format( \
+ b, "%u", ((regs)->i32[(reg)&IREE_I32_REGISTER_MASK]))); \
+ }
+
+static iree_status_t iree_vm_bytecode_disasm_emit_type_name(
+ const iree_vm_type_def_t* type_def, iree_string_builder_t* b) {
+ if (iree_vm_type_def_is_value(type_def)) {
+ const char* type_name;
+ switch (type_def->value_type) {
+ case IREE_VM_VALUE_TYPE_I8:
+ type_name = "i8";
+ break;
+ case IREE_VM_VALUE_TYPE_I16:
+ type_name = "i16";
+ break;
+ case IREE_VM_VALUE_TYPE_I32:
+ type_name = "i32";
+ break;
+ case IREE_VM_VALUE_TYPE_I64:
+ type_name = "i64";
+ break;
+ case IREE_VM_VALUE_TYPE_F32:
+ type_name = "f32";
+ break;
+ case IREE_VM_VALUE_TYPE_F64:
+ type_name = "f64";
+ break;
+ default:
+ type_name = "unknown";
+ break;
+ }
+ return iree_string_builder_append_cstring(b, type_name);
+ } else if (iree_vm_type_def_is_ref(type_def)) {
+ iree_string_view_t type_name = iree_vm_ref_type_name(type_def->ref_type);
+ return iree_string_builder_append_format(b, "%.*s", (int)type_name.size,
+ type_name.data);
+ } else {
+ return iree_string_builder_append_cstring(b, "*");
+ }
+}
+#define EMIT_TYPE_NAME(type_def) \
+ iree_vm_bytecode_disasm_emit_type_name(type_def, b);
+
+static iree_status_t iree_vm_bytecode_disasm_emit_operand_list(
+ const iree_vm_registers_t* regs, const iree_vm_register_list_t* list,
+ iree_vm_bytecode_disasm_format_t format, iree_string_builder_t* b) {
+ bool include_values =
+ regs && (format & IREE_VM_BYTECODE_DISASM_FORMAT_INLINE_VALUES);
+ for (uint16_t i = 0; i < list->size; ++i) {
+ if (i > 0) {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ }
+ uint16_t reg = list->registers[i];
+ EMIT_REG_NAME(reg);
+ if (include_values) {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "("));
+ EMIT_REG_VALUE(regs, reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ }
+ }
+ return iree_ok_status();
+}
+#define EMIT_OPERAND_REG_LIST(reg_list) \
+ iree_vm_bytecode_disasm_emit_operand_list(regs, reg_list, format, b)
+static iree_status_t iree_vm_bytecode_disasm_emit_result_list(
+ const iree_vm_register_list_t* list,
+ iree_vm_bytecode_disasm_format_t format, iree_string_builder_t* b) {
+ for (uint16_t i = 0; i < list->size; ++i) {
+ if (i > 0) {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ }
+ uint16_t reg = list->registers[i];
+ EMIT_REG_NAME(reg);
+ }
+ return iree_ok_status();
+}
+#define EMIT_RESULT_REG_LIST(reg_list) \
+ iree_vm_bytecode_disasm_emit_result_list(reg_list, format, b)
+static iree_status_t iree_vm_bytecode_disasm_emit_remap_list(
+ const iree_vm_registers_t* regs,
+ const iree_vm_register_remap_list_t* remap_list,
+ iree_vm_bytecode_disasm_format_t format, iree_string_builder_t* b) {
+ bool include_values =
+ regs && (format & IREE_VM_BYTECODE_DISASM_FORMAT_INLINE_VALUES);
+ for (uint16_t i = 0; i < remap_list->size; ++i) {
+ if (i > 0) {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ }
+ EMIT_REG_NAME(remap_list->pairs[i].src_reg);
+ if (include_values) {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "("));
+ EMIT_REG_VALUE(regs, remap_list->pairs[i].src_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ }
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "->"));
+ EMIT_REG_NAME(remap_list->pairs[i].dst_reg);
+ }
+ return iree_ok_status();
+}
+#define EMIT_REMAP_LIST(remap_list) \
+ iree_vm_bytecode_disasm_emit_remap_list(regs, remap_list, format, b)
+
+#define EMIT_OPTIONAL_VALUE_I32(expr) \
+ if (regs && (format & IREE_VM_BYTECODE_DISASM_FORMAT_INLINE_VALUES)) { \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(b, "(%" PRId32 ")", \
+ (int32_t)(expr))); \
+ }
+#define EMIT_OPTIONAL_VALUE_I64(expr) \
+ if (regs && (format & IREE_VM_BYTECODE_DISASM_FORMAT_INLINE_VALUES)) { \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format( \
+ b, "(%" PRId64 ")", *(int64_t*)&(expr))); \
+ }
+#define EMIT_OPTIONAL_VALUE_F32(expr) \
+ if (regs && (format & IREE_VM_BYTECODE_DISASM_FORMAT_INLINE_VALUES)) { \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, "(%f)", *(float*)&(expr))); \
+ }
+#define EMIT_OPTIONAL_VALUE_F64(expr) \
+ if (regs && (format & IREE_VM_BYTECODE_DISASM_FORMAT_INLINE_VALUES)) { \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, "(%f)", *(double*)&(expr))); \
+ }
+#define EMIT_OPTIONAL_VALUE_REF(expr) \
+ if (regs && (format & IREE_VM_BYTECODE_DISASM_FORMAT_INLINE_VALUES)) { \
+ iree_vm_ref_t* ref = (expr); \
+ if (iree_vm_ref_is_null(ref)) { \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "(null)")); \
+ } else { \
+ iree_string_view_t type_name = iree_vm_ref_type_name(ref->type); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format( \
+ b, "(!%.*s/0x%p)", (int)type_name.size, type_name.data, ref->ptr)); \
+ } \
+ }
+
+#define DISASM_OP_CORE_UNARY_I32(op_name, op_mnemonic) \
+ DISASM_OP(CORE, op_name) { \
+ uint16_t operand_reg = VM_ParseOperandRegI32("operand"); \
+ uint16_t result_reg = VM_ParseResultRegI32("result"); \
+ EMIT_I32_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_I32_REG_NAME(operand_reg); \
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[operand_reg]); \
+ break; \
+ }
+
+#define DISASM_OP_CORE_BINARY_I32(op_name, op_mnemonic) \
+ DISASM_OP(CORE, op_name) { \
+ uint16_t lhs_reg = VM_ParseOperandRegI32("lhs"); \
+ uint16_t rhs_reg = VM_ParseOperandRegI32("rhs"); \
+ uint16_t result_reg = VM_ParseResultRegI32("result"); \
+ EMIT_I32_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_I32_REG_NAME(lhs_reg); \
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[lhs_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_I32_REG_NAME(rhs_reg); \
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[rhs_reg]); \
+ break; \
+ }
+
+#define DISASM_OP_CORE_TERNARY_I32(op_name, op_mnemonic) \
+ DISASM_OP(CORE, op_name) { \
+ uint16_t a_reg = VM_ParseOperandRegI32("a"); \
+ uint16_t b_reg = VM_ParseOperandRegI32("b"); \
+ uint16_t c_reg = VM_ParseOperandRegI32("c"); \
+ uint16_t result_reg = VM_ParseResultRegI32("result"); \
+ EMIT_I32_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_I32_REG_NAME(a_reg); \
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[a_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_I32_REG_NAME(b_reg); \
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[b_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_I32_REG_NAME(c_reg); \
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[c_reg]); \
+ break; \
+ }
+
+#define DISASM_OP_EXT_I64_UNARY_I64(op_name, op_mnemonic) \
+ DISASM_OP(EXT_I64, op_name) { \
+ uint16_t operand_reg = VM_ParseOperandRegI64("operand"); \
+ uint16_t result_reg = VM_ParseResultRegI64("result"); \
+ EMIT_I64_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_I64_REG_NAME(operand_reg); \
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[operand_reg]); \
+ break; \
+ }
+
+#define DISASM_OP_EXT_I64_BINARY_I64(op_name, op_mnemonic) \
+ DISASM_OP(EXT_I64, op_name) { \
+ uint16_t lhs_reg = VM_ParseOperandRegI64("lhs"); \
+ uint16_t rhs_reg = VM_ParseOperandRegI64("rhs"); \
+ uint16_t result_reg = VM_ParseResultRegI64("result"); \
+ EMIT_I64_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_I64_REG_NAME(lhs_reg); \
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[lhs_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_I64_REG_NAME(rhs_reg); \
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[rhs_reg]); \
+ break; \
+ }
+
+#define DISASM_OP_EXT_I64_TERNARY_I64(op_name, op_mnemonic) \
+ DISASM_OP(EXT_I64, op_name) { \
+ uint16_t a_reg = VM_ParseOperandRegI64("a"); \
+ uint16_t b_reg = VM_ParseOperandRegI64("b"); \
+ uint16_t c_reg = VM_ParseOperandRegI64("c"); \
+ uint16_t result_reg = VM_ParseResultRegI64("result"); \
+ EMIT_I64_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_I64_REG_NAME(a_reg); \
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[a_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_I64_REG_NAME(b_reg); \
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[b_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_I64_REG_NAME(c_reg); \
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[c_reg]); \
+ break; \
+ }
+
+#define DISASM_OP_EXT_F32_UNARY_F32(op_name, op_mnemonic) \
+ DISASM_OP(EXT_F32, op_name) { \
+ uint16_t operand_reg = VM_ParseOperandRegF32("operand"); \
+ uint16_t result_reg = VM_ParseResultRegF32("result"); \
+ EMIT_F32_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_F32_REG_NAME(operand_reg); \
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[operand_reg]); \
+ break; \
+ }
+
+#define DISASM_OP_EXT_F32_BINARY_F32(op_name, op_mnemonic) \
+ DISASM_OP(EXT_F32, op_name) { \
+ uint16_t lhs_reg = VM_ParseOperandRegF32("lhs"); \
+ uint16_t rhs_reg = VM_ParseOperandRegF32("rhs"); \
+ uint16_t result_reg = VM_ParseResultRegF32("result"); \
+ EMIT_F32_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_F32_REG_NAME(lhs_reg); \
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[lhs_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_F32_REG_NAME(rhs_reg); \
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[rhs_reg]); \
+ break; \
+ }
+
+#define DISASM_OP_EXT_F32_TERNARY_F32(op_name, op_mnemonic) \
+ DISASM_OP(EXT_F32, op_name) { \
+ uint16_t a_reg = VM_ParseOperandRegF32("a"); \
+ uint16_t b_reg = VM_ParseOperandRegF32("b"); \
+ uint16_t c_reg = VM_ParseOperandRegF32("c"); \
+ uint16_t result_reg = VM_ParseResultRegF32("result"); \
+ EMIT_F32_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_F32_REG_NAME(a_reg); \
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[a_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_F32_REG_NAME(b_reg); \
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[b_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_F32_REG_NAME(c_reg); \
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[c_reg]); \
+ break; \
+ }
+
+iree_status_t iree_vm_bytecode_disasm_op(
+ iree_vm_bytecode_module_t* module,
+ iree_vm_bytecode_module_state_t* module_state, uint16_t function_ordinal,
+ iree_vm_source_offset_t pc, const iree_vm_registers_t* regs,
+ iree_vm_bytecode_disasm_format_t format, iree_string_builder_t* b) {
+ const uint8_t* IREE_RESTRICT bytecode_data =
+ module->bytecode_data.data +
+ module->function_descriptor_table[function_ordinal].bytecode_offset;
+
+ switch (bytecode_data[pc++]) {
+ //===------------------------------------------------------------------===//
+ // Globals
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP(CORE, GlobalLoadI32) {
+ uint32_t byte_offset = VM_ParseGlobalAttr("global");
+ uint16_t value_reg = VM_ParseResultRegI32("value");
+ EMIT_I32_REG_NAME(value_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, " = vm.global.load.i32 .rwdata[%u]", byte_offset));
+ EMIT_OPTIONAL_VALUE_I32(
+ vm_global_load_i32(module_state->rwdata_storage.data, byte_offset));
+ break;
+ }
+
+ DISASM_OP(CORE, GlobalStoreI32) {
+ uint32_t byte_offset = VM_ParseGlobalAttr("global");
+ uint16_t value_reg = VM_ParseOperandRegI32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, "vm.global.store.i32 "));
+ EMIT_I32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, ", .rwdata[%u]", byte_offset));
+ break;
+ }
+
+ DISASM_OP(CORE, GlobalLoadIndirectI32) {
+ uint16_t byte_offset_reg = VM_ParseOperandRegI32("global");
+ uint16_t value_reg = VM_ParseResultRegI32("value");
+ EMIT_I32_REG_NAME(value_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(
+ b, " = vm.global.load.indirect.i32 .rwdata["));
+ EMIT_I32_REG_NAME(byte_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[byte_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "]"));
+ EMIT_OPTIONAL_VALUE_I32(vm_global_load_i32(
+ module_state->rwdata_storage.data, regs->i32[byte_offset_reg]));
+ break;
+ }
+
+ DISASM_OP(CORE, GlobalStoreIndirectI32) {
+ uint16_t byte_offset_reg = VM_ParseOperandRegI32("global");
+ uint16_t value_reg = VM_ParseOperandRegI32("value");
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(
+ b, "vm.global.store.indirect.i32 "));
+ EMIT_I32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", .rwdata["));
+ EMIT_I32_REG_NAME(byte_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[byte_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "]"));
+ break;
+ }
+
+ DISASM_OP(CORE, GlobalLoadRef) {
+ uint32_t global = VM_ParseGlobalAttr("global");
+ const iree_vm_type_def_t* type_def = VM_ParseTypeOf("value");
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("value", &result_is_move);
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, " = vm.global.load.ref .refs[%u]", global));
+ EMIT_OPTIONAL_VALUE_REF(&module_state->global_ref_table[global]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " : !"));
+ EMIT_TYPE_NAME(type_def);
+ break;
+ }
+
+ DISASM_OP(CORE, GlobalStoreRef) {
+ uint32_t global = VM_ParseGlobalAttr("global");
+ const iree_vm_type_def_t* type_def = VM_ParseTypeOf("value");
+ bool value_is_move;
+ uint16_t value_reg = VM_ParseOperandRegRef("value", &value_is_move);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.global.store.ref "));
+ EMIT_REF_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[value_reg]);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, ", .refs[%u] : !", global));
+ EMIT_TYPE_NAME(type_def);
+ break;
+ }
+
+ DISASM_OP(CORE, GlobalLoadIndirectRef) {
+ uint16_t global_reg = VM_ParseOperandRegI32("global");
+ const iree_vm_type_def_t* type_def = VM_ParseTypeOf("value");
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("value", &result_is_move);
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(
+ b, " = vm.global.load.indirect.ref .refs["));
+ EMIT_I32_REG_NAME(global_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[global_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "]"));
+ EMIT_OPTIONAL_VALUE_REF(
+ &module_state->global_ref_table[regs->i32[global_reg]]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " : !"));
+ EMIT_TYPE_NAME(type_def);
+ break;
+ }
+
+ DISASM_OP(CORE, GlobalStoreIndirectRef) {
+ uint16_t global_reg = VM_ParseOperandRegI32("global");
+ const iree_vm_type_def_t* type_def = VM_ParseTypeOf("value");
+ bool value_is_move;
+ uint16_t value_reg = VM_ParseOperandRegRef("value", &value_is_move);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, "vm.global.store.indirect.ref "));
+ EMIT_REF_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(b, ", .refs["));
+ EMIT_I32_REG_NAME(global_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[global_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(b, "] : !"));
+ EMIT_TYPE_NAME(type_def);
+ break;
+ }
+
+ //===------------------------------------------------------------------===//
+ // Constants
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP(CORE, ConstI32) {
+ int32_t value = VM_ParseIntAttr32("value");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, " = vm.const.i32 %d // 0x%08X", value, value));
+ break;
+ }
+
+ DISASM_OP(CORE, ConstI32Zero) {
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.const.i32.zero"));
+ break;
+ }
+
+ DISASM_OP(CORE, ConstRefZero) {
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move);
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.const.ref.zero"));
+ break;
+ }
+
+ DISASM_OP(CORE, ConstRefRodata) {
+ uint32_t rodata_ordinal = VM_ParseRodataAttr("rodata");
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("value", &result_is_move);
+ iree_vm_buffer_t* buffer =
+ &module_state->rodata_ref_table[rodata_ordinal];
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, " = vm.const.ref.rodata %u // 0x%p %" PRIhsz "b", rodata_ordinal,
+ buffer->data.data, buffer->data.data_length));
+ break;
+ }
+
+ //===------------------------------------------------------------------===//
+ // Buffers
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP(CORE, BufferAlloc) {
+ uint16_t length_reg = VM_ParseOperandRegI32("length");
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move);
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.alloc "));
+ EMIT_I32_REG_NAME(length_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[length_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, BufferClone) {
+ bool source_is_move;
+ uint16_t source_reg = VM_ParseOperandRegRef("source", &source_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("offset");
+ uint16_t length_reg = VM_ParseOperandRegI32("length");
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move);
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.clone "));
+ EMIT_REF_REG_NAME(source_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[source_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(length_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[length_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, BufferLength) {
+ bool buffer_is_move;
+ uint16_t buffer_reg = VM_ParseOperandRegRef("buffer", &buffer_is_move);
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.length "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, BufferCopy) {
+ bool source_buffer_is_move;
+ uint16_t source_buffer_reg =
+ VM_ParseOperandRegRef("source_buffer", &source_buffer_is_move);
+ uint16_t source_offset_reg = VM_ParseOperandRegI32("source_offset");
+ bool target_buffer_is_move;
+ uint16_t target_buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &target_buffer_is_move);
+ uint16_t target_offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t length_reg = VM_ParseOperandRegI32("length");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.copy "));
+ EMIT_REF_REG_NAME(source_buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[source_buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(source_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[source_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(target_buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[target_buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(target_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[target_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(length_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[length_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, BufferCompare) {
+ bool lhs_buffer_is_move;
+ uint16_t lhs_buffer_reg =
+ VM_ParseOperandRegRef("lhs_buffer", &lhs_buffer_is_move);
+ uint16_t lhs_offset_reg = VM_ParseOperandRegI32("lhs_offset");
+ bool rhs_buffer_is_move;
+ uint16_t rhs_buffer_reg =
+ VM_ParseOperandRegRef("rhs_buffer", &rhs_buffer_is_move);
+ uint16_t rhs_offset_reg = VM_ParseOperandRegI32("rhs_offset");
+ uint16_t length_reg = VM_ParseOperandRegI32("length");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.compare "));
+ EMIT_REF_REG_NAME(lhs_buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[lhs_buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(lhs_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[lhs_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(rhs_buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[rhs_buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(rhs_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[rhs_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(length_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[length_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, BufferFillI8) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t length_reg = VM_ParseOperandRegI32("length");
+ uint16_t value_reg = VM_ParseOperandRegI32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.fill.i8 "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(length_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[length_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I32((uint8_t)regs->i32[value_reg]);
+ break;
+ }
+ DISASM_OP(CORE, BufferFillI16) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t length_reg = VM_ParseOperandRegI32("length");
+ uint16_t value_reg = VM_ParseOperandRegI32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.fill.i16 "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint16_t));
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(length_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[length_reg] / sizeof(uint16_t));
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I32((uint16_t)regs->i32[value_reg]);
+ break;
+ }
+ DISASM_OP(CORE, BufferFillI32) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t length_reg = VM_ParseOperandRegI32("length");
+ uint16_t value_reg = VM_ParseOperandRegI32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.fill.i32 "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint32_t));
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(length_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[length_reg] / sizeof(uint32_t));
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[value_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, BufferLoadI8U) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("source_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("source_offset");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.load.i8.u "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg]);
+ break;
+ }
+ DISASM_OP(CORE, BufferLoadI8S) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("source_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("source_offset");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.load.i8.s "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg]);
+ break;
+ }
+ DISASM_OP(CORE, BufferLoadI16U) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("source_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("source_offset");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.load.i16.u "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint16_t));
+ break;
+ }
+ DISASM_OP(CORE, BufferLoadI16S) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("source_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("source_offset");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.load.i16.s "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint16_t));
+ break;
+ }
+ DISASM_OP(CORE, BufferLoadI32) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("source_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("source_offset");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.load.i32 "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint32_t));
+ break;
+ }
+
+ DISASM_OP(CORE, BufferStoreI8) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t value_reg = VM_ParseOperandRegI32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.store.i8 "));
+ EMIT_I32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I32((uint8_t)regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg]);
+ break;
+ }
+ DISASM_OP(CORE, BufferStoreI16) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t value_reg = VM_ParseOperandRegI32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.store.i16 "));
+ EMIT_I32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I32((uint16_t)regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint16_t));
+ break;
+ }
+ DISASM_OP(CORE, BufferStoreI32) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t value_reg = VM_ParseOperandRegI32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.store.i32 "));
+ EMIT_I32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint32_t));
+ break;
+ }
+
+ //===------------------------------------------------------------------===//
+ // Lists
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP(CORE, ListAlloc) {
+ const iree_vm_type_def_t* element_type_def =
+ VM_ParseTypeOf("element_type");
+ uint16_t initial_capacity_reg = VM_ParseOperandRegI32("initial_capacity");
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move);
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.list.alloc "));
+ EMIT_I32_REG_NAME(initial_capacity_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[initial_capacity_reg]);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " : !vm.list<"));
+ EMIT_TYPE_NAME(element_type_def);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ">"));
+ break;
+ }
+
+ DISASM_OP(CORE, ListReserve) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t minimum_capacity_reg = VM_ParseOperandRegI32("minimum_capacity");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.list.reserve "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(minimum_capacity_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[minimum_capacity_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, ListSize) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.list.size "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, ListResize) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t new_size_reg = VM_ParseOperandRegI32("new_size");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.list.resize "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(new_size_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[new_size_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, ListGetI32) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.list.get.i32 "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, ListSetI32) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ uint16_t raw_value_reg = VM_ParseOperandRegI32("raw_value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.list.set.i32 "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(raw_value_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[raw_value_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, ListGetRef) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ const iree_vm_type_def_t* type_def = VM_ParseTypeOf("result");
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move);
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.list.get.ref "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, ListSetRef) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ bool operand_is_move;
+ uint16_t operand_reg = VM_ParseOperandRegRef("value", &operand_is_move);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.list.set.ref "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[operand_reg]);
+ break;
+ }
+
+ //===------------------------------------------------------------------===//
+ // Conditional assignment
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP(CORE, SelectI32) {
+ uint16_t condition_reg = VM_ParseOperandRegI32("condition");
+ uint16_t true_value_reg = VM_ParseOperandRegI32("true_value");
+ uint16_t false_value_reg = VM_ParseOperandRegI32("false_value");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.select.i32 "));
+ EMIT_I32_REG_NAME(condition_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[condition_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " ? "));
+ EMIT_I32_REG_NAME(true_value_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[true_value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " : "));
+ EMIT_I32_REG_NAME(false_value_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[false_value_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, SelectRef) {
+ uint16_t condition_reg = VM_ParseOperandRegI32("condition");
+ const iree_vm_type_def_t* type_def = VM_ParseTypeOf("true_value");
+ bool true_value_is_move;
+ uint16_t true_value_reg =
+ VM_ParseOperandRegRef("true_value", &true_value_is_move);
+ bool false_value_is_move;
+ uint16_t false_value_reg =
+ VM_ParseOperandRegRef("false_value", &false_value_is_move);
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move);
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.select.ref "));
+ EMIT_I32_REG_NAME(condition_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[condition_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " ? "));
+ EMIT_REF_REG_NAME(true_value_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[true_value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " : "));
+ EMIT_REF_REG_NAME(false_value_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[false_value_reg]);
+ break;
+ }
+
+ DISASM_OP(CORE, SwitchI32) {
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ int32_t default_value = VM_ParseIntAttr32("default_value");
+ const iree_vm_register_list_t* value_reg_list =
+ VM_ParseVariadicOperands("values");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.switch.i32 "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "["));
+ EMIT_OPERAND_REG_LIST(value_reg_list);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, "] else %u", default_value));
+ break;
+ }
+
+ DISASM_OP(CORE, SwitchRef) {
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ bool default_is_move;
+ uint16_t default_value_reg =
+ VM_ParseOperandRegRef("default_value", &default_is_move);
+ const iree_vm_register_list_t* value_reg_list =
+ VM_ParseVariadicOperands("values");
+ bool result_is_move;
+ uint16_t result_reg = VM_ParseResultRegRef("result", &result_is_move);
+ EMIT_REF_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.switch.ref "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "["));
+ EMIT_OPERAND_REG_LIST(value_reg_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "] else "));
+ EMIT_REF_REG_NAME(default_value_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[default_value_reg]);
+ break;
+ }
+
+ //===------------------------------------------------------------------===//
+ // Native integer arithmetic
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP_CORE_BINARY_I32(AddI32, "vm.add.i32");
+ DISASM_OP_CORE_BINARY_I32(SubI32, "vm.sub.i32");
+ DISASM_OP_CORE_BINARY_I32(MulI32, "vm.mul.i32");
+ DISASM_OP_CORE_BINARY_I32(DivI32S, "vm.div.i32.s");
+ DISASM_OP_CORE_BINARY_I32(DivI32U, "vm.div.i32.u");
+ DISASM_OP_CORE_BINARY_I32(RemI32S, "vm.rem.i32.s");
+ DISASM_OP_CORE_BINARY_I32(RemI32U, "vm.rem.i32.u");
+ DISASM_OP_CORE_TERNARY_I32(FMAI32, "vm.fma.i32");
+ DISASM_OP_CORE_UNARY_I32(NotI32, "vm.not.i32");
+ DISASM_OP_CORE_BINARY_I32(AndI32, "vm.and.i32");
+ DISASM_OP_CORE_BINARY_I32(OrI32, "vm.or.i32");
+ DISASM_OP_CORE_BINARY_I32(XorI32, "vm.xor.i32");
+
+ //===------------------------------------------------------------------===//
+ // Casting and type conversion/emulation
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP_CORE_UNARY_I32(TruncI32I8, "vm.trunc.i32.i8");
+ DISASM_OP_CORE_UNARY_I32(TruncI32I16, "vm.trunc.i32.i16");
+ DISASM_OP_CORE_UNARY_I32(ExtI8I32S, "vm.ext.i8.i32.s");
+ DISASM_OP_CORE_UNARY_I32(ExtI8I32U, "vm.ext.i8.i32.u");
+ DISASM_OP_CORE_UNARY_I32(ExtI16I32S, "vm.ext.i16.i32.s");
+ DISASM_OP_CORE_UNARY_I32(ExtI16I32U, "vm.ext.i16.i32.u");
+
+ //===------------------------------------------------------------------===//
+ // Native bitwise shifts and rotates
+ //===------------------------------------------------------------------===//
+
+#define DISASM_OP_CORE_SHIFT_I32(op_name, op_mnemonic) \
+ DISASM_OP(CORE, op_name) { \
+ uint16_t operand_reg = VM_ParseOperandRegI32("operand"); \
+ uint16_t amount_reg = VM_ParseOperandRegI32("amount"); \
+ uint16_t result_reg = VM_ParseResultRegI32("result"); \
+ EMIT_I32_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_I32_REG_NAME(operand_reg); \
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[operand_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_I32_REG_NAME(amount_reg); \
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[amount_reg]); \
+ break; \
+ }
+
+ DISASM_OP_CORE_SHIFT_I32(ShlI32, "vm.shl.i32");
+ DISASM_OP_CORE_SHIFT_I32(ShrI32S, "vm.shr.i32.s");
+ DISASM_OP_CORE_SHIFT_I32(ShrI32U, "vm.shr.i32.u");
+
+ //===------------------------------------------------------------------===//
+ // Comparison ops
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP_CORE_BINARY_I32(CmpEQI32, "vm.cmp.eq.i32");
+ DISASM_OP_CORE_BINARY_I32(CmpNEI32, "vm.cmp.ne.i32");
+ DISASM_OP_CORE_BINARY_I32(CmpLTI32S, "vm.cmp.lt.i32.s");
+ DISASM_OP_CORE_BINARY_I32(CmpLTI32U, "vm.cmp.lt.i32.u");
+ DISASM_OP_CORE_UNARY_I32(CmpNZI32, "vm.cmp.nz.i32");
+
+ DISASM_OP(CORE, CmpEQRef) {
+ bool lhs_is_move;
+ uint16_t lhs_reg = VM_ParseOperandRegRef("lhs", &lhs_is_move);
+ bool rhs_is_move;
+ uint16_t rhs_reg = VM_ParseOperandRegRef("rhs", &rhs_is_move);
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.cmp.eq.ref "));
+ EMIT_REF_REG_NAME(lhs_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[lhs_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(rhs_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[rhs_reg]);
+ break;
+ }
+ DISASM_OP(CORE, CmpNERef) {
+ bool lhs_is_move;
+ uint16_t lhs_reg = VM_ParseOperandRegRef("lhs", &lhs_is_move);
+ bool rhs_is_move;
+ uint16_t rhs_reg = VM_ParseOperandRegRef("rhs", &rhs_is_move);
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.cmp.ne.ref "));
+ EMIT_REF_REG_NAME(lhs_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[lhs_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(rhs_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[rhs_reg]);
+ break;
+ }
+ DISASM_OP(CORE, CmpNZRef) {
+ bool operand_is_move;
+ uint16_t operand_reg = VM_ParseOperandRegRef("operand", &operand_is_move);
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.cmp.nz.ref "));
+ EMIT_REF_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[operand_reg]);
+ break;
+ }
+
+ //===------------------------------------------------------------------===//
+ // Control flow
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP(CORE, Branch) {
+ int32_t block_pc = VM_ParseBranchTarget("dest");
+ const iree_vm_register_remap_list_t* remap_list =
+ VM_ParseBranchOperands("operands");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, "vm.br ^%08X(", block_pc));
+ EMIT_REMAP_LIST(remap_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ break;
+ }
+
+ DISASM_OP(CORE, CondBranch) {
+ uint16_t condition_reg = VM_ParseOperandRegI32("condition");
+ int32_t true_block_pc = VM_ParseBranchTarget("true_dest");
+ const iree_vm_register_remap_list_t* true_remap_list =
+ VM_ParseBranchOperands("true_operands");
+ int32_t false_block_pc = VM_ParseBranchTarget("false_dest");
+ const iree_vm_register_remap_list_t* false_remap_list =
+ VM_ParseBranchOperands("false_operands");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.cond_br "));
+ EMIT_I32_REG_NAME(condition_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[condition_reg]);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, ", ^%08X(", true_block_pc));
+ EMIT_REMAP_LIST(true_remap_list);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, "), ^%08X(", false_block_pc));
+ EMIT_REMAP_LIST(false_remap_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ break;
+ }
+
+ DISASM_OP(CORE, Call) {
+ int32_t function_ordinal = VM_ParseFuncAttr("callee");
+ const iree_vm_register_list_t* src_reg_list =
+ VM_ParseVariadicOperands("operands");
+ const iree_vm_register_list_t* dst_reg_list =
+ VM_ParseVariadicResults("results");
+ if (dst_reg_list->size > 0) {
+ EMIT_RESULT_REG_LIST(dst_reg_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " = "));
+ }
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "vm.call @"));
+ int is_import = (function_ordinal & 0x80000000u) != 0;
+ iree_vm_function_t function;
+ if (is_import) {
+ const iree_vm_bytecode_import_t* import =
+ &module_state->import_table[function_ordinal & 0x7FFFFFFFu];
+ function = import->function;
+ } else {
+ function.module = &module->interface;
+ function.linkage = IREE_VM_FUNCTION_LINKAGE_INTERNAL;
+ function.ordinal = function_ordinal;
+ }
+ iree_string_view_t module_name = iree_vm_module_name(function.module);
+ iree_string_view_t func_name = iree_vm_function_name(&function);
+ if (iree_string_view_is_empty(func_name)) {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, "%.*s:%u", (int)module_name.size, module_name.data,
+ function.ordinal));
+ } else {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, "%.*s.%.*s", (int)module_name.size, module_name.data,
+ (int)func_name.size, func_name.data));
+ }
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "("));
+ EMIT_OPERAND_REG_LIST(src_reg_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ break;
+ }
+
+ DISASM_OP(CORE, CallVariadic) {
+ int32_t function_ordinal = VM_ParseFuncAttr("callee");
+ // TODO(benvanik): print segment sizes.
+ // const iree_vm_register_list_t* segment_size_list =
+ VM_ParseVariadicOperands("segment_sizes");
+ const iree_vm_register_list_t* src_reg_list =
+ VM_ParseVariadicOperands("operands");
+ const iree_vm_register_list_t* dst_reg_list =
+ VM_ParseVariadicResults("results");
+ if (dst_reg_list->size > 0) {
+ EMIT_RESULT_REG_LIST(dst_reg_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " = "));
+ }
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.call.varadic @"));
+ int is_import = (function_ordinal & 0x80000000u) != 0;
+ iree_vm_function_t function;
+ if (is_import) {
+ const iree_vm_bytecode_import_t* import =
+ &module_state->import_table[function_ordinal & 0x7FFFFFFFu];
+ function = import->function;
+ } else {
+ function.module = &module->interface;
+ function.linkage = IREE_VM_FUNCTION_LINKAGE_INTERNAL;
+ function.ordinal = function_ordinal;
+ }
+ iree_string_view_t module_name = iree_vm_module_name(function.module);
+ iree_string_view_t func_name = iree_vm_function_name(&function);
+ if (iree_string_view_is_empty(func_name)) {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, "%.*s:%u", (int)module_name.size, module_name.data,
+ function.ordinal));
+ } else {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, "%.*s.%.*s", (int)module_name.size, module_name.data,
+ (int)func_name.size, func_name.data));
+ }
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "("));
+ EMIT_OPERAND_REG_LIST(src_reg_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ break;
+ }
+
+ DISASM_OP(CORE, Return) {
+ const iree_vm_register_list_t* src_reg_list =
+ VM_ParseVariadicOperands("operands");
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "vm.return "));
+ EMIT_OPERAND_REG_LIST(src_reg_list);
+ break;
+ }
+
+ DISASM_OP(CORE, Fail) {
+ uint16_t status_code_reg = VM_ParseOperandRegI32("status");
+ iree_string_view_t message;
+ VM_ParseStrAttr("message", &message);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "vm.fail "));
+ EMIT_I32_REG_NAME(status_code_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[status_code_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, ", \"%.*s\"", (int)message.size, message.data));
+ break;
+ }
+
+ //===------------------------------------------------------------------===//
+ // Async/fiber ops
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP(CORE, Yield) {
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.yield (TBD)"));
+ break;
+ }
+
+ //===------------------------------------------------------------------===//
+ // Debugging
+ //===------------------------------------------------------------------===//
+
+ DISASM_OP(CORE, Trace) {
+ iree_string_view_t event_name;
+ VM_ParseStrAttr("event_name", &event_name);
+ const iree_vm_register_list_t* src_reg_list =
+ VM_ParseVariadicOperands("operands");
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, "vm.trace \"%.*s\"(", (int)event_name.size, event_name.data));
+ EMIT_OPERAND_REG_LIST(src_reg_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ break;
+ }
+
+ DISASM_OP(CORE, Print) {
+ iree_string_view_t event_name;
+ VM_ParseStrAttr("event_name", &event_name);
+ const iree_vm_register_list_t* src_reg_list =
+ VM_ParseVariadicOperands("operands");
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, "vm.print \"%.*s\"(", (int)event_name.size, event_name.data));
+ EMIT_OPERAND_REG_LIST(src_reg_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ break;
+ }
+
+ DISASM_OP(CORE, Break) {
+ int32_t block_pc = VM_DecBranchTarget("dest");
+ const iree_vm_register_remap_list_t* remap_list =
+ VM_ParseBranchOperands("operands");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, "vm.break ^%08X(", block_pc));
+ EMIT_REMAP_LIST(remap_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ break;
+ }
+
+ DISASM_OP(CORE, CondBreak) {
+ uint16_t condition_reg = VM_ParseOperandRegI32("condition");
+ int32_t block_pc = VM_ParseBranchTarget("dest");
+ const iree_vm_register_remap_list_t* remap_list =
+ VM_ParseBranchOperands("operands");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.cond_break "));
+ EMIT_I32_REG_NAME(condition_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[condition_reg]);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, ", ^%08X(", block_pc));
+ EMIT_REMAP_LIST(remap_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ")"));
+ break;
+ }
+
+ //===------------------------------------------------------------------===//
+ // Extension trampolines
+ //===------------------------------------------------------------------===//
+
+#if IREE_VM_EXT_I64_ENABLE
+ BEGIN_DISASM_PREFIX(PrefixExtI64, EXT_I64)
+
+ //===----------------------------------------------------------------===//
+ // ExtI64: Globals
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_I64, GlobalLoadI64) {
+ uint32_t byte_offset = VM_ParseGlobalAttr("global");
+ uint16_t value_reg = VM_ParseResultRegI64("value");
+ EMIT_I32_REG_NAME(value_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, " = vm.global.load.i64 .rwdata[%u]", byte_offset));
+ EMIT_OPTIONAL_VALUE_I64(module_state->rwdata_storage.data[byte_offset]);
+ break;
+ }
+
+ DISASM_OP(EXT_I64, GlobalStoreI64) {
+ uint32_t byte_offset = VM_ParseGlobalAttr("global");
+ uint16_t value_reg = VM_ParseOperandRegI64("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, "vm.global.store.i64 "));
+ EMIT_I64_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, ", .rwdata[%u]", byte_offset));
+ break;
+ }
+
+ DISASM_OP(EXT_I64, GlobalLoadIndirectI64) {
+ uint16_t byte_offset_reg = VM_ParseOperandRegI32("global");
+ uint16_t value_reg = VM_ParseResultRegI64("value");
+ EMIT_I64_REG_NAME(value_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(
+ b, " = vm.global.load.indirect.i64 .rwdata["));
+ EMIT_I32_REG_NAME(byte_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[byte_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "]"));
+ EMIT_OPTIONAL_VALUE_I64(
+ module_state->rwdata_storage.data[regs->i32[byte_offset_reg]]);
+ break;
+ }
+
+ DISASM_OP(EXT_I64, GlobalStoreIndirectI64) {
+ uint16_t byte_offset_reg = VM_ParseOperandRegI32("global");
+ uint16_t value_reg = VM_ParseOperandRegI64("value");
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(
+ b, "vm.global.store.indirect.i64 "));
+ EMIT_I64_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", .rwdata["));
+ EMIT_I32_REG_NAME(byte_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[byte_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "]"));
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtI64: Constants
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_I64, ConstI64) {
+ int64_t value = VM_ParseIntAttr64("value");
+ uint16_t result_reg = VM_ParseResultRegI64("result");
+ EMIT_I64_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, " = vm.const.i64 %" PRId64 " // 0x%08" PRIX64 "", value, value));
+ break;
+ }
+
+ DISASM_OP(EXT_I64, ConstI64Zero) {
+ uint16_t result_reg = VM_ParseResultRegI64("result");
+ EMIT_I64_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.const.i64.zero"));
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtI64: Lists
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_I64, ListGetI64) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ uint16_t result_reg = VM_ParseResultRegI64("result");
+ EMIT_I64_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.list.get.i64 "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ break;
+ }
+
+ DISASM_OP(EXT_I64, ListSetI64) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ uint16_t value_reg = VM_ParseOperandRegI64("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.list.set.i64 "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I64_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[value_reg]);
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtI64: Conditional assignment
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_I64, SelectI64) {
+ uint16_t condition_reg = VM_ParseOperandRegI32("condition");
+ uint16_t true_value_reg = VM_ParseOperandRegI64("true_value");
+ uint16_t false_value_reg = VM_ParseOperandRegI64("false_value");
+ uint16_t result_reg = VM_ParseResultRegI64("result");
+ EMIT_I64_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.select.i64 "));
+ EMIT_I32_REG_NAME(condition_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[condition_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " ? "));
+ EMIT_I64_REG_NAME(true_value_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[true_value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " : "));
+ EMIT_I64_REG_NAME(false_value_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[false_value_reg]);
+ break;
+ }
+
+ DISASM_OP(EXT_I64, SwitchI64) {
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ int64_t default_value = VM_ParseIntAttr64("default_value");
+ const iree_vm_register_list_t* value_reg_list =
+ VM_ParseVariadicOperands("values");
+ uint16_t result_reg = VM_ParseResultRegI64("result");
+ EMIT_I64_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.switch.i64 "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "["));
+ EMIT_OPERAND_REG_LIST(value_reg_list);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, "] else %" PRId64, default_value));
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtI64: Native integer arithmetic
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP_EXT_I64_BINARY_I64(AddI64, "vm.add.i64");
+ DISASM_OP_EXT_I64_BINARY_I64(SubI64, "vm.sub.i64");
+ DISASM_OP_EXT_I64_BINARY_I64(MulI64, "vm.mul.i64");
+ DISASM_OP_EXT_I64_BINARY_I64(DivI64S, "vm.div.i64.s");
+ DISASM_OP_EXT_I64_BINARY_I64(DivI64U, "vm.div.i64.u");
+ DISASM_OP_EXT_I64_BINARY_I64(RemI64S, "vm.rem.i64.s");
+ DISASM_OP_EXT_I64_BINARY_I64(RemI64U, "vm.rem.i64.u");
+ DISASM_OP_EXT_I64_TERNARY_I64(FMAI64, "vm.fma.i64");
+ DISASM_OP_EXT_I64_UNARY_I64(NotI64, "vm.not.i64");
+ DISASM_OP_EXT_I64_BINARY_I64(AndI64, "vm.and.i64");
+ DISASM_OP_EXT_I64_BINARY_I64(OrI64, "vm.or.i64");
+ DISASM_OP_EXT_I64_BINARY_I64(XorI64, "vm.xor.i64");
+
+ //===----------------------------------------------------------------===//
+ // ExtI64: Casting and type conversion/emulation
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_I64, TruncI64I32) {
+ uint16_t operand_reg = VM_ParseOperandRegI64("operand");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.trunc.i64.i32 "));
+ EMIT_I64_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[operand_reg]);
+ break;
+ }
+ DISASM_OP(EXT_I64, ExtI32I64S) {
+ uint16_t operand_reg = VM_ParseOperandRegI32("operand");
+ uint16_t result_reg = VM_ParseResultRegI64("result");
+ EMIT_I64_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.ext.i32.i64.s "));
+ EMIT_I64_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[operand_reg]);
+ break;
+ }
+ DISASM_OP(EXT_I64, ExtI32I64U) {
+ uint16_t operand_reg = VM_ParseOperandRegI32("operand");
+ uint16_t result_reg = VM_ParseResultRegI64("result");
+ EMIT_I64_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.ext.i32.i64.u "));
+ EMIT_I64_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[operand_reg]);
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtI64: Native bitwise shifts and rotates
+ //===----------------------------------------------------------------===//
+
+#define DISASM_OP_EXT_I64_SHIFT_I64(op_name, op_mnemonic) \
+ DISASM_OP(EXT_I64, op_name) { \
+ uint16_t operand_reg = VM_ParseOperandRegI64("operand"); \
+ uint16_t amount_reg = VM_ParseOperandRegI32("amount"); \
+ uint16_t result_reg = VM_ParseResultRegI64("result"); \
+ EMIT_I64_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_I64_REG_NAME(operand_reg); \
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[operand_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_I32_REG_NAME(amount_reg); \
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[amount_reg]); \
+ break; \
+ }
+
+ DISASM_OP_EXT_I64_SHIFT_I64(ShlI64, "vm.shl.i64");
+ DISASM_OP_EXT_I64_SHIFT_I64(ShrI64S, "vm.shr.i64.s");
+ DISASM_OP_EXT_I64_SHIFT_I64(ShrI64U, "vm.shr.i64.u");
+
+ //===----------------------------------------------------------------===//
+ // ExtI64: Comparison ops
+ //===----------------------------------------------------------------===//
+
+#define DISASM_OP_EXT_I64_CMP_I64(op_name, op_mnemonic) \
+ DISASM_OP(EXT_I64, op_name) { \
+ uint16_t lhs_reg = VM_ParseOperandRegI64("lhs"); \
+ uint16_t rhs_reg = VM_ParseOperandRegI64("rhs"); \
+ uint16_t result_reg = VM_ParseResultRegI32("result"); \
+ EMIT_I32_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_I64_REG_NAME(lhs_reg); \
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[lhs_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_I64_REG_NAME(rhs_reg); \
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[rhs_reg]); \
+ break; \
+ }
+
+ DISASM_OP_EXT_I64_CMP_I64(CmpEQI64, "vm.cmp.eq.i64");
+ DISASM_OP_EXT_I64_CMP_I64(CmpNEI64, "vm.cmp.ne.i64");
+ DISASM_OP_EXT_I64_CMP_I64(CmpLTI64S, "vm.cmp.lt.i64.s");
+ DISASM_OP_EXT_I64_CMP_I64(CmpLTI64U, "vm.cmp.lt.i64.u");
+ DISASM_OP(EXT_I64, CmpNZI64) {
+ uint16_t operand_reg = VM_ParseOperandRegI64("operand");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.cmp.nz.i64 "));
+ EMIT_I64_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[operand_reg]);
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtI64: Buffers
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_I64, BufferFillI64) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t length_reg = VM_ParseOperandRegI32("length");
+ uint16_t value_reg = VM_ParseOperandRegI64("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.fill.i64 "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint64_t));
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(length_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[length_reg] / sizeof(uint64_t));
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I64_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[value_reg]);
+ break;
+ }
+
+ DISASM_OP(EXT_I64, BufferLoadI64) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("source_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("source_offset");
+ uint16_t result_reg = VM_ParseResultRegI64("result");
+ EMIT_I64_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.load.i64 "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint64_t));
+ break;
+ }
+
+ DISASM_OP(EXT_I64, BufferStoreI64) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t value_reg = VM_ParseOperandRegI64("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.store.i64 "));
+ EMIT_I64_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_I64(regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint32_t));
+ break;
+ }
+
+ END_DISASM_PREFIX()
+#else
+ UNHANDLED_DISASM_PREFIX(PrefixExtI64, EXT_I64);
+#endif // IREE_VM_EXT_I64_ENABLE
+
+#if IREE_VM_EXT_F32_ENABLE
+ BEGIN_DISASM_PREFIX(PrefixExtF32, EXT_F32)
+
+ //===----------------------------------------------------------------===//
+ // ExtF32: Globals
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_F32, GlobalLoadF32) {
+ uint32_t byte_offset = VM_ParseGlobalAttr("global");
+ uint16_t value_reg = VM_ParseResultRegF32("value");
+ EMIT_F32_REG_NAME(value_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ b, " = vm.global.load.f32 .rwdata[%u]", byte_offset));
+ EMIT_OPTIONAL_VALUE_F32(module_state->rwdata_storage.data[byte_offset]);
+ break;
+ }
+
+ DISASM_OP(EXT_F32, GlobalStoreF32) {
+ uint32_t byte_offset = VM_ParseGlobalAttr("global");
+ uint16_t value_reg = VM_ParseOperandRegF32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, "vm.global.store.f32 "));
+ EMIT_F32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, ", .rwdata[%u]", byte_offset));
+ break;
+ }
+
+ DISASM_OP(EXT_F32, GlobalLoadIndirectF32) {
+ uint16_t byte_offset_reg = VM_ParseOperandRegI32("global");
+ uint16_t value_reg = VM_ParseResultRegI32("value");
+ EMIT_F32_REG_NAME(value_reg);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(
+ b, " = vm.global.load.indirect.f32 .rwdata["));
+ EMIT_I32_REG_NAME(byte_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[byte_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "]"));
+ EMIT_OPTIONAL_VALUE_F32(
+ module_state->rwdata_storage.data[regs->i32[byte_offset_reg]]);
+ break;
+ }
+
+ DISASM_OP(EXT_F32, GlobalStoreIndirectF32) {
+ uint16_t byte_offset_reg = VM_ParseOperandRegI32("global");
+ uint16_t value_reg = VM_ParseOperandRegF32("value");
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(
+ b, "vm.global.store.indirect.f32 "));
+ EMIT_F32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", .rwdata["));
+ EMIT_I32_REG_NAME(byte_offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[byte_offset_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "]"));
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtF32: Constants
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_F32, ConstF32) {
+ float value = VM_ParseFloatAttr32("value");
+ uint16_t result_reg = VM_ParseResultRegF32("result");
+ EMIT_F32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, " = vm.const.f32 %f", value));
+ break;
+ }
+
+ DISASM_OP(EXT_F32, ConstF32Zero) {
+ uint16_t result_reg = VM_ParseResultRegF32("result");
+ EMIT_F32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.const.f32.zero"));
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtF32: Lists
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_F32, ListGetF32) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ uint16_t result_reg = VM_ParseResultRegF32("result");
+ EMIT_F32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.list.get.f32 "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ break;
+ }
+
+ DISASM_OP(EXT_F32, ListSetF32) {
+ bool list_is_move;
+ uint16_t list_reg = VM_ParseOperandRegRef("list", &list_is_move);
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ uint16_t raw_value_reg = VM_ParseOperandRegF32("raw_value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.list.set.f32 "));
+ EMIT_REF_REG_NAME(list_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[list_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_F32_REG_NAME(raw_value_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[raw_value_reg]);
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtF32: Conditional assignment
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_F32, SelectF32) {
+ uint16_t condition_reg = VM_ParseOperandRegI32("condition");
+ uint16_t true_value_reg = VM_ParseOperandRegF32("true_value");
+ uint16_t false_value_reg = VM_ParseOperandRegF32("false_value");
+ uint16_t result_reg = VM_ParseResultRegF32("result");
+ EMIT_F32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.select.f32 "));
+ EMIT_I32_REG_NAME(condition_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[condition_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " ? "));
+ EMIT_F32_REG_NAME(true_value_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[true_value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, " : "));
+ EMIT_F32_REG_NAME(false_value_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[false_value_reg]);
+ break;
+ }
+
+ DISASM_OP(EXT_F32, SwitchF32) {
+ uint16_t index_reg = VM_ParseOperandRegI32("index");
+ float default_value = VM_ParseFloatAttr32("default_value");
+ const iree_vm_register_list_t* value_reg_list =
+ VM_ParseVariadicOperands("values");
+ uint16_t result_reg = VM_ParseResultRegF32("result");
+ EMIT_F32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.switch.f32 "));
+ EMIT_I32_REG_NAME(index_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[index_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, "["));
+ EMIT_OPERAND_REG_LIST(value_reg_list);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_format(b, "] else %f", default_value));
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtF32: Native floating-point arithmetic
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP_EXT_F32_BINARY_F32(AddF32, "vm.add.f32");
+ DISASM_OP_EXT_F32_BINARY_F32(SubF32, "vm.sub.f32");
+ DISASM_OP_EXT_F32_BINARY_F32(MulF32, "vm.mul.f32");
+ DISASM_OP_EXT_F32_BINARY_F32(DivF32, "vm.div.f32");
+ DISASM_OP_EXT_F32_BINARY_F32(RemF32, "vm.rem.f32");
+ DISASM_OP_EXT_F32_TERNARY_F32(FMAF32, "vm.fma.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(AbsF32, "vm.abs.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(NegF32, "vm.neg.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(CeilF32, "vm.ceil.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(FloorF32, "vm.floor.f32");
+
+ DISASM_OP_EXT_F32_UNARY_F32(AtanF32, "vm.atan.f32");
+ DISASM_OP_EXT_F32_BINARY_F32(Atan2F32, "vm.atan2.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(CosF32, "vm.cos.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(SinF32, "vm.sin.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(ExpF32, "vm.exp.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(Exp2F32, "vm.exp2.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(ExpM1F32, "vm.expm1.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(LogF32, "vm.log.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(Log10F32, "vm.log10.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(Log1pF32, "vm.log1p.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(Log2F32, "vm.log2.f32");
+ DISASM_OP_EXT_F32_BINARY_F32(PowF32, "vm.pow.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(RsqrtF32, "vm.rsqrt.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(SqrtF32, "vm.sqrt.f32");
+ DISASM_OP_EXT_F32_UNARY_F32(TanhF32, "vm.tanh.f32");
+
+ //===----------------------------------------------------------------===//
+ // ExtF32: Casting and type conversion/emulation
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_F32, CastSI32F32) {
+ uint16_t operand_reg = VM_ParseOperandRegI32("operand");
+ uint16_t result_reg = VM_ParseResultRegF32("result");
+ EMIT_F32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.cast.si32.f32 "));
+ EMIT_I32_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[operand_reg]);
+ break;
+ }
+ DISASM_OP(EXT_F32, CastUI32F32) {
+ uint16_t operand_reg = VM_ParseOperandRegI32("operand");
+ uint16_t result_reg = VM_ParseResultRegF32("result");
+ EMIT_F32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.cast.ui32.f32 "));
+ EMIT_I32_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[operand_reg]);
+ break;
+ }
+ DISASM_OP(EXT_F32, CastF32SI32) {
+ uint16_t operand_reg = VM_ParseOperandRegF32("operand");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.cast.f32.sif32 "));
+ EMIT_F32_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[operand_reg]);
+ break;
+ }
+ DISASM_OP(EXT_F32, CastF32UI32) {
+ uint16_t operand_reg = VM_ParseOperandRegF32("operand");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.cast.f32.uif32 "));
+ EMIT_F32_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[operand_reg]);
+ break;
+ }
+ DISASM_OP(EXT_F32, BitcastI32F32) {
+ uint16_t operand_reg = VM_ParseOperandRegI32("operand");
+ uint16_t result_reg = VM_ParseResultRegF32("result");
+ EMIT_F32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.bitcast.i32.f32 "));
+ EMIT_I32_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[operand_reg]);
+ break;
+ }
+ DISASM_OP(EXT_F32, BitcastF32I32) {
+ uint16_t operand_reg = VM_ParseOperandRegF32("operand");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.bitcast.f32.if32 "));
+ EMIT_F32_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[operand_reg]);
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtF32: Comparison ops
+ //===----------------------------------------------------------------===//
+
+#define DISASM_OP_EXT_F32_CMP_F32(op_name, op_mnemonic) \
+ DISASM_OP(EXT_F32, op_name) { \
+ uint16_t lhs_reg = VM_ParseOperandRegF32("lhs"); \
+ uint16_t rhs_reg = VM_ParseOperandRegF32("rhs"); \
+ uint16_t result_reg = VM_ParseResultRegI32("result"); \
+ EMIT_I32_REG_NAME(result_reg); \
+ IREE_RETURN_IF_ERROR( \
+ iree_string_builder_append_format(b, " = %s ", op_mnemonic)); \
+ EMIT_F32_REG_NAME(lhs_reg); \
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[lhs_reg]); \
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", ")); \
+ EMIT_F32_REG_NAME(rhs_reg); \
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[rhs_reg]); \
+ break; \
+ }
+
+ DISASM_OP_EXT_F32_CMP_F32(CmpEQF32O, "vm.cmp.eq.f32.o");
+ DISASM_OP_EXT_F32_CMP_F32(CmpEQF32U, "vm.cmp.eq.f32.u");
+ DISASM_OP_EXT_F32_CMP_F32(CmpNEF32O, "vm.cmp.ne.f32.o");
+ DISASM_OP_EXT_F32_CMP_F32(CmpNEF32U, "vm.cmp.ne.f32.u");
+ DISASM_OP_EXT_F32_CMP_F32(CmpLTF32O, "vm.cmp.lt.f32.o");
+ DISASM_OP_EXT_F32_CMP_F32(CmpLTF32U, "vm.cmp.lt.f32.u");
+ DISASM_OP_EXT_F32_CMP_F32(CmpLTEF32O, "vm.cmp.lte.f32.o");
+ DISASM_OP_EXT_F32_CMP_F32(CmpLTEF32U, "vm.cmp.lte.f32.u");
+ DISASM_OP(EXT_F32, CmpNaNF32) {
+ uint16_t operand_reg = VM_ParseOperandRegF32("operand");
+ uint16_t result_reg = VM_ParseResultRegI32("result");
+ EMIT_I32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.cmp.nan.f32 "));
+ EMIT_F32_REG_NAME(operand_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[operand_reg]);
+ break;
+ }
+
+ //===----------------------------------------------------------------===//
+ // ExtF32: Buffers
+ //===----------------------------------------------------------------===//
+
+ DISASM_OP(EXT_F32, BufferFillF32) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t length_reg = VM_ParseOperandRegI32("length");
+ uint16_t value_reg = VM_ParseOperandRegF32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.fill.f32 "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(float));
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(length_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[length_reg] / sizeof(float));
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_F32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[value_reg]);
+ break;
+ }
+
+ DISASM_OP(EXT_F32, BufferLoadF32) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("source_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("source_offset");
+ uint16_t result_reg = VM_ParseResultRegF32("result");
+ EMIT_F32_REG_NAME(result_reg);
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, " = vm.buffer.load.f32 "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(float));
+ break;
+ }
+
+ DISASM_OP(EXT_F32, BufferStoreF32) {
+ bool buffer_is_move;
+ uint16_t buffer_reg =
+ VM_ParseOperandRegRef("target_buffer", &buffer_is_move);
+ uint16_t offset_reg = VM_ParseOperandRegI32("target_offset");
+ uint16_t value_reg = VM_ParseOperandRegF32("value");
+ IREE_RETURN_IF_ERROR(
+ iree_string_builder_append_cstring(b, "vm.buffer.store.f32 "));
+ EMIT_F32_REG_NAME(value_reg);
+ EMIT_OPTIONAL_VALUE_F32(regs->i32[value_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_REF_REG_NAME(buffer_reg);
+ EMIT_OPTIONAL_VALUE_REF(®s->ref[buffer_reg]);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(b, ", "));
+ EMIT_I32_REG_NAME(offset_reg);
+ EMIT_OPTIONAL_VALUE_I32(regs->i32[offset_reg] / sizeof(uint32_t));
+ break;
+ }
+
+ END_DISASM_PREFIX()
+#else
+ UNHANDLED_DISASM_PREFIX(PrefixExtF32, EXT_F32)
+#endif // IREE_VM_EXT_F32_ENABLE
+ UNHANDLED_DISASM_PREFIX(PrefixExtF64, EXT_F64)
+
+ default:
+ return iree_make_status(IREE_STATUS_UNIMPLEMENTED,
+ "unhandled core opcode");
+ }
+ return iree_ok_status();
+}
+
+iree_status_t iree_vm_bytecode_trace_disasm(iree_vm_stack_frame_t* frame,
+ iree_vm_source_offset_t pc,
+ const iree_vm_registers_t* regs,
+ FILE* file) {
+ iree_string_builder_t b;
+ iree_string_builder_initialize(iree_allocator_system(), &b);
+
+ // TODO(benvanik): ensure frame is in-sync before call or restore original.
+ // It's shady to manipulate the frame here but I know we expect the pc to be
+ // valid only on entry/exit from a function.
+ frame->pc = pc;
+
+#if IREE_VM_EXECUTION_TRACING_SRC_LOC_ENABLE
+ iree_vm_source_location_t source_location;
+ iree_status_t status = iree_vm_module_resolve_source_location(
+ frame->function.module, frame, &source_location);
+ if (iree_status_is_ok(status)) {
+ status = iree_vm_source_location_format(
+ &source_location, IREE_VM_SOURCE_LOCATION_FORMAT_FLAG_SINGLE_LINE, &b);
+ }
+ if (iree_status_is_ok(status)) {
+ // Pad out to keep alignment. This is just guesswork based on my machine.
+ static const iree_host_size_t pad_to = 80;
+ iree_host_size_t col = iree_string_builder_size(&b);
+ if (col < pad_to) {
+ iree_string_builder_append_format(&b, "%*s ", (int)(pad_to - col), "");
+ } else {
+ status = iree_string_builder_append_cstring(&b, " ");
+ }
+ } else {
+ // Ignore failures when no source location is available.
+ if (iree_status_is_unavailable(status)) {
+ status = iree_ok_status();
+ } else {
+ return status;
+ }
+ }
+#else
+ iree_status_t status = iree_ok_status();
+#endif // IREE_VM_EXECUTION_TRACING_ENABLE
+
+ if (iree_status_is_ok(status)) {
+ iree_string_view_t module_name =
+ iree_vm_module_name(frame->function.module);
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ &b, "[%.*s", (int)module_name.size, module_name.data));
+ iree_string_view_t function_name = iree_vm_function_name(&frame->function);
+ if (iree_string_view_is_empty(function_name)) {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ &b, "@%u", (uint32_t)frame->function.ordinal));
+ } else {
+ IREE_RETURN_IF_ERROR(iree_string_builder_append_format(
+ &b, ".%.*s", (int)function_name.size, function_name.data));
+ }
+ status = iree_string_builder_append_format(&b, "+%08" PRIX64 "] ", pc);
+ }
+
+ if (iree_status_is_ok(status)) {
+ status = iree_vm_bytecode_disasm_op(
+ (iree_vm_bytecode_module_t*)frame->function.module,
+ (iree_vm_bytecode_module_state_t*)frame->module_state,
+ frame->function.ordinal, pc, regs,
+ IREE_VM_BYTECODE_DISASM_FORMAT_INLINE_VALUES, &b);
+ }
+
+ if (iree_status_is_ok(status)) {
+ fprintf(file, "%.*s\n", (int)iree_string_builder_size(&b),
+ iree_string_builder_buffer(&b));
+ }
+
+ iree_string_builder_deinitialize(&b);
+ return status;
+}
diff --git a/iree/vm/bytecode_disasm.h b/iree/vm/bytecode_disasm.h
new file mode 100644
index 0000000..2c73025
--- /dev/null
+++ b/iree/vm/bytecode_disasm.h
@@ -0,0 +1,46 @@
+// Copyright 2021 The IREE Authors
+//
+// Licensed under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+
+#ifndef IREE_VM_BYTECODE_DISASM_H_
+#define IREE_VM_BYTECODE_DISASM_H_
+
+#include <stdio.h>
+
+#include "iree/base/string_builder.h"
+#include "iree/vm/bytecode_dispatch_util.h"
+#include "iree/vm/bytecode_module_impl.h"
+#include "iree/vm/stack.h"
+
+// Controls how bytecode disassembly is formatted.
+typedef enum iree_vm_bytecode_disasm_format_e {
+ IREE_VM_BYTECODE_DISASM_FORMAT_DEFAULT = 0,
+ // Includes the input register values inline in the op text.
+ // Example: `%i0 <= ShrI32U %i2(5), %i3(6)`
+ IREE_VM_BYTECODE_DISASM_FORMAT_INLINE_VALUES = 1u << 0,
+} iree_vm_bytecode_disasm_format_t;
+
+// Disassembles the bytecode operation at |pc| using the provided module state.
+// Appends the disasembled op to |string_builder| in a format based on |format|.
+// If |regs| are available then values can be added using the format mode.
+//
+// Example: `%i0 <= ShrI32U %i2, %i3`
+//
+// WARNING: this does not currently perform any verification on the bytecode;
+// it's assumed all bytecode is valid. This is a debug tool: you shouldn't be
+// running this in production on untrusted inputs anyway.
+iree_status_t iree_vm_bytecode_disasm_op(
+ iree_vm_bytecode_module_t* module,
+ iree_vm_bytecode_module_state_t* module_state, uint16_t function_ordinal,
+ iree_vm_source_offset_t pc, const iree_vm_registers_t* regs,
+ iree_vm_bytecode_disasm_format_t format,
+ iree_string_builder_t* string_builder);
+
+iree_status_t iree_vm_bytecode_trace_disasm(iree_vm_stack_frame_t* frame,
+ iree_vm_source_offset_t pc,
+ const iree_vm_registers_t* regs,
+ FILE* file);
+
+#endif // IREE_VM_BYTECODE_DISASM_H_
diff --git a/iree/vm/bytecode_dispatch.c b/iree/vm/bytecode_dispatch.c
index 43e61fa..23c7647 100644
--- a/iree/vm/bytecode_dispatch.c
+++ b/iree/vm/bytecode_dispatch.c
@@ -11,6 +11,7 @@
#include "iree/base/api.h"
#include "iree/base/internal/math.h"
#include "iree/vm/api.h"
+#include "iree/vm/bytecode_disasm.h"
#include "iree/vm/bytecode_dispatch_util.h"
#include "iree/vm/bytecode_module_impl.h"
#include "iree/vm/ops.h"
@@ -104,8 +105,6 @@
iree_vm_stack_t* stack, const iree_vm_function_t function,
iree_vm_stack_frame_t** out_callee_frame,
iree_vm_registers_t* out_callee_registers) {
- IREE_DISPATCH_LOG_CALL(&function);
-
iree_vm_bytecode_module_t* module =
(iree_vm_bytecode_module_t*)function.module->self;
if (IREE_UNLIKELY(function.ordinal >= module->function_descriptor_count)) {
@@ -542,7 +541,6 @@
iree_vm_function_call_t call;
memset(&call, 0, sizeof(call));
call.function = import->function;
- IREE_DISPATCH_LOG_CALL(&call.function);
// Marshal inputs from registers to the ABI arguments buffer.
call.arguments.data_length = import->argument_buffer_size;
@@ -584,7 +582,6 @@
iree_vm_function_call_t call;
memset(&call, 0, sizeof(call));
call.function = import->function;
- IREE_DISPATCH_LOG_CALL(&call.function);
// Allocate ABI argument/result storage taking into account the variadic
// segments.
@@ -742,7 +739,7 @@
});
DISPATCH_OP(CORE, GlobalLoadIndirectRef, {
- uint32_t global = VM_DecGlobalAttr("global");
+ uint32_t global = VM_DecOperandRegI32("global");
if (IREE_UNLIKELY(global >= module_state->global_ref_count)) {
return iree_make_status(
IREE_STATUS_OUT_OF_RANGE,
@@ -758,7 +755,7 @@
});
DISPATCH_OP(CORE, GlobalStoreIndirectRef, {
- uint32_t global = VM_DecGlobalAttr("global");
+ uint32_t global = VM_DecOperandRegI32("global");
if (IREE_UNLIKELY(global >= module_state->global_ref_count)) {
return iree_make_status(
IREE_STATUS_OUT_OF_RANGE,
diff --git a/iree/vm/bytecode_dispatch_test.cc b/iree/vm/bytecode_dispatch_test.cc
index dc7dcf4..77ab994 100644
--- a/iree/vm/bytecode_dispatch_test.cc
+++ b/iree/vm/bytecode_dispatch_test.cc
@@ -82,8 +82,8 @@
std::vector<iree_vm_module_t*> modules = {bytecode_module_};
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- instance_, modules.data(), modules.size(), iree_allocator_system(),
- &context_));
+ instance_, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &context_));
}
virtual void TearDown() {
@@ -98,7 +98,7 @@
bytecode_module_->self, IREE_VM_FUNCTION_LINKAGE_EXPORT,
iree_make_cstring_view(function_name), &function));
- return iree_vm_invoke(context_, function,
+ return iree_vm_invoke(context_, function, IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/nullptr, /*inputs=*/nullptr,
/*outputs=*/nullptr, iree_allocator_system());
}
diff --git a/iree/vm/bytecode_dispatch_util.h b/iree/vm/bytecode_dispatch_util.h
index 5e17867..676b342 100644
--- a/iree/vm/bytecode_dispatch_util.h
+++ b/iree/vm/bytecode_dispatch_util.h
@@ -118,20 +118,24 @@
// Debugging utilities
//===----------------------------------------------------------------------===//
-// Enable to get some verbose logging; better than nothing until we have some
-// better tooling.
-#define IREE_DISPATCH_LOGGING 0
-
-#if IREE_DISPATCH_LOGGING
-#include <stdio.h>
-#define IREE_DISPATCH_LOG_OPCODE(op_name) \
- fprintf(stderr, "DISPATCH %d %s\n", (int)pc, op_name)
-#define IREE_DISPATCH_LOG_CALL(target_function) \
- fprintf(stderr, "CALL -> %s\n", iree_vm_function_name(target_function).data);
+#if IREE_VM_EXECUTION_TRACING_FORCE_ENABLE
+#define IREE_IS_DISPATCH_TRACING_ENABLED() true
#else
-#define IREE_DISPATCH_LOG_OPCODE(...)
-#define IREE_DISPATCH_LOG_CALL(...)
-#endif // IREE_DISPATCH_LOGGING
+#define IREE_IS_DISPATCH_TRACING_ENABLED() \
+ !!(iree_vm_stack_invocation_flags(stack) & \
+ IREE_VM_INVOCATION_FLAG_TRACE_EXECUTION)
+#endif // IREE_VM_EXECUTION_TRACING_FORCE_ENABLE
+
+#if IREE_VM_EXECUTION_TRACING_ENABLE
+#define IREE_DISPATCH_TRACE_INSTRUCTION(pc_offset, op_name) \
+ if (IREE_IS_DISPATCH_TRACING_ENABLED()) { \
+ IREE_RETURN_IF_ERROR(iree_vm_bytecode_trace_disasm( \
+ current_frame, (pc - (pc_offset)), ®s, stderr)); \
+ }
+
+#else
+#define IREE_DISPATCH_TRACE_INSTRUCTION(...)
+#endif // IREE_VM_EXECUTION_TRACING_ENABLE
#if defined(IREE_COMPILER_MSVC) && !defined(IREE_COMPILER_CLANG)
#define IREE_DISPATCH_MODE_SWITCH 1
@@ -266,6 +270,14 @@
// doesn't support it, though, and there may be other targets (like wasm) that
// can only handle the switch-based approach.
+// Bytecode data -offset used when looking for the start of the currently
+// dispatched instruction: `instruction_start = pc - OFFSET`
+#define VM_PC_OFFSET_CORE 1
+#define VM_PC_OFFSET_EXT_I32 2
+#define VM_PC_OFFSET_EXT_I64 2
+#define VM_PC_OFFSET_EXT_F32 2
+#define VM_PC_OFFSET_EXT_F64 2
+
#if defined(IREE_DISPATCH_MODE_COMPUTED_GOTO)
// Dispatch table mapping 1:1 with bytecode ops.
@@ -336,9 +348,10 @@
"unhandled dispatch extension " #ext); \
}
-#define DISPATCH_OP(ext, op_name, body) \
- _dispatch_##ext##_##op_name : IREE_DISPATCH_LOG_OPCODE(#op_name); \
- body; \
+#define DISPATCH_OP(ext, op_name, body) \
+ _dispatch_##ext##_##op_name:; \
+ IREE_DISPATCH_TRACE_INSTRUCTION(VM_PC_OFFSET_##ext, #op_name); \
+ body; \
goto* kDispatchTable_CORE[bytecode_data[pc++]];
#define BEGIN_DISPATCH_PREFIX(op_name, ext) \
@@ -371,10 +384,10 @@
"unhandled dispatch extension " #ext); \
}
-#define DISPATCH_OP(ext, op_name, body) \
- case IREE_VM_OP_##ext##_##op_name: { \
- IREE_DISPATCH_LOG_OPCODE(#op_name); \
- body; \
+#define DISPATCH_OP(ext, op_name, body) \
+ case IREE_VM_OP_##ext##_##op_name: { \
+ IREE_DISPATCH_TRACE_INSTRUCTION(VM_PC_OFFSET_##ext, #op_name); \
+ body; \
} break;
#define BEGIN_DISPATCH_PREFIX(op_name, ext) \
diff --git a/iree/vm/bytecode_module.c b/iree/vm/bytecode_module.c
index 9f3370e..af15527 100644
--- a/iree/vm/bytecode_module.c
+++ b/iree/vm/bytecode_module.c
@@ -230,7 +230,7 @@
uint16_t* out_ordinal,
iree_vm_FunctionSignatureDef_table_t* out_signature_def) {
*out_ordinal = 0;
- *out_signature_def = NULL;
+ if (out_signature_def) *out_signature_def = NULL;
uint16_t ordinal = function.ordinal;
iree_vm_FunctionSignatureDef_table_t signature_def = NULL;
@@ -262,7 +262,7 @@
}
*out_ordinal = ordinal;
- *out_signature_def = signature_def;
+ if (out_signature_def) *out_signature_def = signature_def;
return iree_ok_status();
}
@@ -474,6 +474,7 @@
static iree_status_t iree_vm_bytecode_location_format(
int32_t location_ordinal,
iree_vm_LocationTypeDef_union_vec_t location_table,
+ iree_vm_source_location_format_flags_t flags,
iree_string_builder_t* builder) {
iree_vm_LocationTypeDef_union_t location =
iree_vm_LocationTypeDef_union_vec_at(location_table, location_ordinal);
@@ -488,11 +489,11 @@
iree_vm_CallSiteLocDef_table_t loc =
(iree_vm_CallSiteLocDef_table_t)location.value;
IREE_RETURN_IF_ERROR(iree_vm_bytecode_location_format(
- iree_vm_CallSiteLocDef_callee(loc), location_table, builder));
+ iree_vm_CallSiteLocDef_callee(loc), location_table, flags, builder));
IREE_RETURN_IF_ERROR(
iree_string_builder_append_cstring(builder, "\n at "));
return iree_vm_bytecode_location_format(
- iree_vm_CallSiteLocDef_caller(loc), location_table, builder);
+ iree_vm_CallSiteLocDef_caller(loc), location_table, flags, builder);
}
case iree_vm_LocationTypeDef_FileLineColLocDef: {
iree_vm_FileLineColLocDef_table_t loc =
@@ -523,7 +524,8 @@
iree_string_builder_append_cstring(builder, ",\n "));
}
IREE_RETURN_IF_ERROR(iree_vm_bytecode_location_format(
- flatbuffers_int32_vec_at(child_locs, i), location_table, builder));
+ flatbuffers_int32_vec_at(child_locs, i), location_table, flags,
+ builder));
}
IREE_RETURN_IF_ERROR(
iree_string_builder_append_cstring(builder, "\n ]"));
@@ -538,7 +540,8 @@
if (iree_vm_NameLocDef_child_location_is_present(loc)) {
IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(builder, "("));
IREE_RETURN_IF_ERROR(iree_vm_bytecode_location_format(
- iree_vm_NameLocDef_child_location(loc), location_table, builder));
+ iree_vm_NameLocDef_child_location(loc), location_table, flags,
+ builder));
IREE_RETURN_IF_ERROR(iree_string_builder_append_cstring(builder, ")"));
}
return iree_ok_status();
@@ -547,7 +550,8 @@
}
static iree_status_t iree_vm_bytecode_module_source_location_format(
- void* self, uint64_t data[2], iree_string_builder_t* builder) {
+ void* self, uint64_t data[2], iree_vm_source_location_format_flags_t flags,
+ iree_string_builder_t* builder) {
iree_vm_DebugDatabaseDef_table_t debug_database_def =
(iree_vm_DebugDatabaseDef_table_t)self;
iree_vm_FunctionSourceMapDef_table_t source_map_def =
@@ -572,7 +576,7 @@
iree_vm_LocationTypeDef_union_vec_t location_table =
iree_vm_DebugDatabaseDef_location_table_union(debug_database_def);
IREE_RETURN_IF_ERROR(iree_vm_bytecode_location_format(
- location_def->location, location_table, builder));
+ location_def->location, location_table, flags, builder));
return iree_ok_status();
}
@@ -591,10 +595,13 @@
// Map the (potentially) export ordinal into the internal function ordinal in
// the function descriptor table.
- uint16_t ordinal = 0;
- iree_vm_FunctionSignatureDef_table_t signature_def = NULL;
- IREE_RETURN_IF_ERROR(iree_vm_bytecode_map_internal_ordinal(
- module, frame->function, &ordinal, &signature_def));
+ uint16_t ordinal;
+ if (frame->function.linkage == IREE_VM_FUNCTION_LINKAGE_INTERNAL) {
+ ordinal = frame->function.ordinal;
+ } else {
+ IREE_RETURN_IF_ERROR(iree_vm_bytecode_map_internal_ordinal(
+ module, frame->function, &ordinal, NULL));
+ }
// Lookup the source map for the function, if available.
iree_vm_FunctionSourceMapDef_vec_t source_maps_vec =
diff --git a/iree/vm/bytecode_module_benchmark.cc b/iree/vm/bytecode_module_benchmark.cc
index ed42af8..9dd7960 100644
--- a/iree/vm/bytecode_module_benchmark.cc
+++ b/iree/vm/bytecode_module_benchmark.cc
@@ -89,8 +89,8 @@
std::array<iree_vm_module_t*, 2> modules = {import_module, bytecode_module};
iree_vm_context_t* context = NULL;
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- instance, modules.data(), modules.size(), iree_allocator_system(),
- &context));
+ instance, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &context));
iree_vm_function_t function;
IREE_CHECK_OK(
@@ -106,8 +106,9 @@
iree_make_byte_span(iree_alloca(result_count * sizeof(int32_t)),
result_count * sizeof(int32_t));
- IREE_VM_INLINE_STACK_INITIALIZE(
- stack, iree_vm_context_state_resolver(context), iree_allocator_system());
+ IREE_VM_INLINE_STACK_INITIALIZE(stack, IREE_VM_INVOCATION_FLAG_NONE,
+ iree_vm_context_state_resolver(context),
+ iree_allocator_system());
while (state.KeepRunningBatch(batch_size)) {
for (iree_host_size_t i = 0; i < i32_args.size(); ++i) {
reinterpret_cast<int32_t*>(call.arguments.data)[i] = i32_args[i];
diff --git a/iree/vm/bytecode_module_size_benchmark.cc b/iree/vm/bytecode_module_size_benchmark.cc
index adcd930..164a223 100644
--- a/iree/vm/bytecode_module_size_benchmark.cc
+++ b/iree/vm/bytecode_module_size_benchmark.cc
@@ -23,7 +23,8 @@
iree_allocator_null(), iree_allocator_system(), &module);
iree_vm_context_t* context = nullptr;
- iree_vm_context_create_with_modules(instance, &module, /*module_count=*/1,
+ iree_vm_context_create_with_modules(instance, IREE_VM_CONTEXT_FLAG_NONE,
+ &module, /*module_count=*/1,
iree_allocator_system(), &context);
iree_vm_function_t function;
@@ -31,7 +32,8 @@
module, IREE_VM_FUNCTION_LINKAGE_EXPORT,
iree_make_cstring_view("empty_func"), &function);
- iree_vm_invoke(context, function, /*policy=*/nullptr, /*inputs=*/nullptr,
+ iree_vm_invoke(context, function, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/nullptr, /*inputs=*/nullptr,
/*outputs=*/nullptr, iree_allocator_system());
iree_vm_module_release(module);
diff --git a/iree/vm/context.c b/iree/vm/context.c
index bbbe798..4fc7938 100644
--- a/iree/vm/context.c
+++ b/iree/vm/context.c
@@ -24,6 +24,9 @@
// Context storage is statically allocated and need not be freed.
uint32_t is_static : 1;
+ // Configuration flags.
+ iree_vm_context_flags_t flags;
+
struct {
iree_host_size_t count;
iree_host_size_t capacity;
@@ -161,7 +164,11 @@
// Run module __deinit functions, if present (in reverse init order).
IREE_VM_INLINE_STACK_INITIALIZE(
- stack, iree_vm_context_state_resolver(context), context->allocator);
+ stack,
+ context->flags & IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION
+ ? IREE_VM_INVOCATION_FLAG_TRACE_EXECUTION
+ : IREE_VM_INVOCATION_FLAG_NONE,
+ iree_vm_context_state_resolver(context), context->allocator);
for (int i = (int)end; i >= (int)start; --i) {
iree_vm_module_t* module = context->list.modules[i];
iree_vm_module_state_t* module_state = context->list.module_states[i];
@@ -195,17 +202,17 @@
IREE_TRACE_ZONE_END(z0);
}
-IREE_API_EXPORT iree_status_t
-iree_vm_context_create(iree_vm_instance_t* instance, iree_allocator_t allocator,
- iree_vm_context_t** out_context) {
- return iree_vm_context_create_with_modules(instance, NULL, 0, allocator,
- out_context);
+IREE_API_EXPORT iree_status_t iree_vm_context_create(
+ iree_vm_instance_t* instance, iree_vm_context_flags_t flags,
+ iree_allocator_t allocator, iree_vm_context_t** out_context) {
+ return iree_vm_context_create_with_modules(instance, flags, NULL, 0,
+ allocator, out_context);
}
IREE_API_EXPORT iree_status_t iree_vm_context_create_with_modules(
- iree_vm_instance_t* instance, iree_vm_module_t** modules,
- iree_host_size_t module_count, iree_allocator_t allocator,
- iree_vm_context_t** out_context) {
+ iree_vm_instance_t* instance, iree_vm_context_flags_t flags,
+ iree_vm_module_t** modules, iree_host_size_t module_count,
+ iree_allocator_t allocator, iree_vm_context_t** out_context) {
IREE_TRACE_ZONE_BEGIN(z0);
IREE_ASSERT_ARGUMENT(out_context);
*out_context = NULL;
@@ -225,6 +232,11 @@
context->context_id = iree_atomic_fetch_add_int32(&next_context_id, 1,
iree_memory_order_seq_cst);
+ // TODO(benvanik): allow for non-frozen but static contexts.
+ context->is_frozen = module_count > 0;
+ context->is_static = module_count > 0;
+ context->flags = flags;
+
uint8_t* p = (uint8_t*)context + sizeof(iree_vm_context_t);
context->list.modules = (iree_vm_module_t**)p;
p += sizeof(iree_vm_module_t*) * module_count;
@@ -232,9 +244,6 @@
p += sizeof(iree_vm_module_state_t*) * module_count;
context->list.count = 0;
context->list.capacity = module_count;
- // TODO(benvanik): allow for non-frozen but static contexts.
- context->is_frozen = module_count > 0;
- context->is_static = module_count > 0;
iree_status_t register_status =
iree_vm_context_register_modules(context, modules, module_count);
@@ -294,6 +303,12 @@
return context->context_id;
}
+IREE_API_EXPORT iree_vm_context_flags_t
+iree_vm_context_flags(const iree_vm_context_t* context) {
+ IREE_ASSERT_ARGUMENT(context);
+ return context->flags;
+}
+
IREE_API_EXPORT iree_status_t iree_vm_context_register_modules(
iree_vm_context_t* context, iree_vm_module_t** modules,
iree_host_size_t module_count) {
@@ -352,7 +367,11 @@
// VM stack used to call into module __init methods.
IREE_VM_INLINE_STACK_INITIALIZE(
- stack, iree_vm_context_state_resolver(context), context->allocator);
+ stack,
+ context->flags & IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION
+ ? IREE_VM_INVOCATION_FLAG_TRACE_EXECUTION
+ : IREE_VM_INVOCATION_FLAG_NONE,
+ iree_vm_context_state_resolver(context), context->allocator);
// Retain all modules and allocate their state.
assert(context->list.capacity >= context->list.count + module_count);
diff --git a/iree/vm/context.h b/iree/vm/context.h
index 0a33f0a..71c36d3 100644
--- a/iree/vm/context.h
+++ b/iree/vm/context.h
@@ -31,11 +31,24 @@
// Thread-compatible and must be externally synchronized.
typedef struct iree_vm_context_t iree_vm_context_t;
+enum iree_vm_context_flag_bits_t {
+ IREE_VM_CONTEXT_FLAG_NONE = 0u,
+
+ // Enables tracing of execution to stderr (when available).
+ // See iree/base/config.h for the flags that control whether this
+ // functionality is available; specifically:
+ // -DIREE_VM_EXECUTION_TRACING_ENABLE=1
+ // All invocations made to this context - including initializers - will be
+ // traced. For fine-grained control use `iree_vm_invocation_flags_t`.
+ IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION = 1u << 0,
+};
+typedef uint32_t iree_vm_context_flags_t;
+
// Creates a new context that uses the given |instance| for device management.
// |out_context| must be released by the caller.
-IREE_API_EXPORT iree_status_t
-iree_vm_context_create(iree_vm_instance_t* instance, iree_allocator_t allocator,
- iree_vm_context_t** out_context);
+IREE_API_EXPORT iree_status_t iree_vm_context_create(
+ iree_vm_instance_t* instance, iree_vm_context_flags_t flags,
+ iree_allocator_t allocator, iree_vm_context_t** out_context);
// Creates a new context with the given static set of modules.
// This is equivalent to iree_vm_context_create+iree_vm_context_register_modules
@@ -43,9 +56,9 @@
// have additional modules registered after creation.
// |out_context| must be released by the caller.
IREE_API_EXPORT iree_status_t iree_vm_context_create_with_modules(
- iree_vm_instance_t* instance, iree_vm_module_t** modules,
- iree_host_size_t module_count, iree_allocator_t allocator,
- iree_vm_context_t** out_context);
+ iree_vm_instance_t* instance, iree_vm_context_flags_t flags,
+ iree_vm_module_t** modules, iree_host_size_t module_count,
+ iree_allocator_t allocator, iree_vm_context_t** out_context);
// Retains the given |context| for the caller.
IREE_API_EXPORT void iree_vm_context_retain(iree_vm_context_t* context);
@@ -56,6 +69,10 @@
// Returns a process-unique ID for the |context|.
IREE_API_EXPORT intptr_t iree_vm_context_id(const iree_vm_context_t* context);
+// Returns |context| flags.
+IREE_API_EXPORT iree_vm_context_flags_t
+iree_vm_context_flags(const iree_vm_context_t* context);
+
// Registers a list of modules with the context and resolves imports in the
// order provided.
// The modules will be retained by the context until destruction.
diff --git a/iree/vm/invocation.c b/iree/vm/invocation.c
index baaea94..58e385c 100644
--- a/iree/vm/invocation.c
+++ b/iree/vm/invocation.c
@@ -201,13 +201,19 @@
IREE_API_EXPORT iree_status_t iree_vm_invoke(
iree_vm_context_t* context, iree_vm_function_t function,
- const iree_vm_invocation_policy_t* policy, iree_vm_list_t* inputs,
- iree_vm_list_t* outputs, iree_allocator_t allocator) {
+ iree_vm_invocation_flags_t flags, const iree_vm_invocation_policy_t* policy,
+ iree_vm_list_t* inputs, iree_vm_list_t* outputs,
+ iree_allocator_t allocator) {
IREE_TRACE_ZONE_BEGIN(z0);
+ // Force tracing if specified on the context.
+ if (iree_vm_context_flags(context) & IREE_VM_CONTEXT_FLAG_TRACE_EXECUTION) {
+ flags |= IREE_VM_INVOCATION_FLAG_TRACE_EXECUTION;
+ }
+
// Allocate a VM stack on the host stack and initialize it.
IREE_VM_INLINE_STACK_INITIALIZE(
- stack, iree_vm_context_state_resolver(context), allocator);
+ stack, flags, iree_vm_context_state_resolver(context), allocator);
iree_status_t status =
iree_vm_invoke_within(context, stack, function, policy, inputs, outputs);
if (!iree_status_is_ok(status)) {
diff --git a/iree/vm/invocation.h b/iree/vm/invocation.h
index cdb026e..9de07b2 100644
--- a/iree/vm/invocation.h
+++ b/iree/vm/invocation.h
@@ -36,14 +36,16 @@
// caller.
IREE_API_EXPORT iree_status_t iree_vm_invoke(
iree_vm_context_t* context, iree_vm_function_t function,
- const iree_vm_invocation_policy_t* policy, iree_vm_list_t* inputs,
- iree_vm_list_t* outputs, iree_allocator_t allocator);
+ iree_vm_invocation_flags_t flags, const iree_vm_invocation_policy_t* policy,
+ iree_vm_list_t* inputs, iree_vm_list_t* outputs,
+ iree_allocator_t allocator);
// TODO(benvanik): document and implement.
IREE_API_EXPORT iree_status_t iree_vm_invocation_create(
iree_vm_context_t* context, iree_vm_function_t function,
- const iree_vm_invocation_policy_t* policy, const iree_vm_list_t* inputs,
- iree_allocator_t allocator, iree_vm_invocation_t** out_invocation);
+ iree_vm_invocation_flags_t flags, const iree_vm_invocation_policy_t* policy,
+ const iree_vm_list_t* inputs, iree_allocator_t allocator,
+ iree_vm_invocation_t** out_invocation);
// Retains the given |invocation| for the caller.
IREE_API_EXPORT iree_status_t
diff --git a/iree/vm/module.c b/iree/vm/module.c
index 9703b5b..144d200 100644
--- a/iree/vm/module.c
+++ b/iree/vm/module.c
@@ -275,13 +275,14 @@
IREE_API_EXPORT iree_status_t
iree_vm_source_location_format(iree_vm_source_location_t* source_location,
+ iree_vm_source_location_format_flags_t flags,
iree_string_builder_t* builder) {
IREE_ASSERT_ARGUMENT(builder);
if (!source_location || !source_location->format) {
return iree_status_from_code(IREE_STATUS_UNAVAILABLE);
}
return source_location->format(source_location->self, source_location->data,
- builder);
+ flags, builder);
}
IREE_API_EXPORT iree_string_view_t
diff --git a/iree/vm/module.h b/iree/vm/module.h
index db7d516..f164527 100644
--- a/iree/vm/module.h
+++ b/iree/vm/module.h
@@ -261,6 +261,15 @@
int reserved;
} iree_vm_execution_result_t;
+// Controls how source locations are formatted into strings.
+enum iree_vm_source_location_format_flag_bits_e {
+ IREE_VM_SOURCE_LOCATION_FORMAT_FLAG_NONE = 0u,
+ // Only formats a single line (excluding \n) for the source location, even
+ // if the full location information (such as a backtrace) is available.
+ IREE_VM_SOURCE_LOCATION_FORMAT_FLAG_SINGLE_LINE = 1u << 0,
+};
+typedef uint32_t iree_vm_source_location_format_flags_t;
+
// Source location interface.
typedef struct iree_vm_source_location_t {
IREE_API_UNSTABLE
@@ -269,13 +278,17 @@
void* self;
uint64_t data[2];
- iree_status_t(IREE_API_PTR* format)(void* self, uint64_t data[2],
- iree_string_builder_t* builder);
+ iree_status_t(IREE_API_PTR* format)(
+ void* self, uint64_t data[2],
+ iree_vm_source_location_format_flags_t flags,
+ iree_string_builder_t* builder);
} iree_vm_source_location_t;
// Formats the |source_location| to its canonical string form.
-IREE_API_EXPORT iree_status_t iree_vm_source_location_format(
- iree_vm_source_location_t* source_location, iree_string_builder_t* builder);
+IREE_API_EXPORT iree_status_t
+iree_vm_source_location_format(iree_vm_source_location_t* source_location,
+ iree_vm_source_location_format_flags_t flags,
+ iree_string_builder_t* builder);
// Defines an interface that can be used to reflect and execute functions on a
// module.
diff --git a/iree/vm/native_module_test.cc b/iree/vm/native_module_test.cc
index c807088..84202d0 100644
--- a/iree/vm/native_module_test.cc
+++ b/iree/vm/native_module_test.cc
@@ -41,8 +41,8 @@
// will be allocated.
std::vector<iree_vm_module_t*> modules = {module_a, module_b};
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- instance_, modules.data(), modules.size(), iree_allocator_system(),
- &context_));
+ instance_, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &context_));
// No longer need the modules as the context retains them.
iree_vm_module_release(module_a);
@@ -77,10 +77,10 @@
/*element_type=*/nullptr, 1, iree_allocator_system(), &output_list));
// Invoke the entry function to do our work. Runs synchronously.
- IREE_RETURN_IF_ERROR(iree_vm_invoke(context_, function,
- /*policy=*/nullptr, input_list.get(),
- output_list.get(),
- iree_allocator_system()));
+ IREE_RETURN_IF_ERROR(
+ iree_vm_invoke(context_, function, IREE_VM_INVOCATION_FLAG_NONE,
+ /*policy=*/nullptr, input_list.get(), output_list.get(),
+ iree_allocator_system()));
// Load the output result.
iree_vm_value_t ret0_value;
diff --git a/iree/vm/stack.c b/iree/vm/stack.c
index dd22155..acb4fb8 100644
--- a/iree/vm/stack.c
+++ b/iree/vm/stack.c
@@ -173,6 +173,9 @@
iree_host_size_t frame_storage_size;
void* frame_storage;
+ // Flags controlling the behavior of the invocation owning this stack.
+ iree_vm_invocation_flags_t flags;
+
// True if the stack owns the frame_storage and should free it when it is no
// longer required. Host stack-allocated stacks don't own their storage but
// may transition to owning it on dynamic growth.
@@ -192,8 +195,9 @@
//===----------------------------------------------------------------------===//
IREE_API_EXPORT iree_status_t iree_vm_stack_initialize(
- iree_byte_span_t storage, iree_vm_state_resolver_t state_resolver,
- iree_allocator_t allocator, iree_vm_stack_t** out_stack) {
+ iree_byte_span_t storage, iree_vm_invocation_flags_t flags,
+ iree_vm_state_resolver_t state_resolver, iree_allocator_t allocator,
+ iree_vm_stack_t** out_stack) {
IREE_ASSERT_ARGUMENT(out_stack);
*out_stack = NULL;
if (storage.data_length < IREE_VM_STACK_MIN_SIZE) {
@@ -208,6 +212,7 @@
iree_vm_stack_t* stack = (iree_vm_stack_t*)storage.data;
memset(stack, 0, sizeof(iree_vm_stack_t));
stack->owns_frame_storage = false;
+ stack->flags = flags;
stack->state_resolver = state_resolver;
stack->allocator = allocator;
@@ -240,8 +245,8 @@
}
IREE_API_EXPORT iree_status_t iree_vm_stack_allocate(
- iree_vm_state_resolver_t state_resolver, iree_allocator_t allocator,
- iree_vm_stack_t** out_stack) {
+ iree_vm_invocation_flags_t flags, iree_vm_state_resolver_t state_resolver,
+ iree_allocator_t allocator, iree_vm_stack_t** out_stack) {
IREE_TRACE_ZONE_BEGIN(z0);
*out_stack = NULL;
@@ -253,8 +258,8 @@
iree_vm_stack_t* stack = NULL;
if (iree_status_is_ok(status)) {
iree_byte_span_t storage_span = iree_make_byte_span(storage, storage_size);
- status = iree_vm_stack_initialize(storage_span, state_resolver, allocator,
- &stack);
+ status = iree_vm_stack_initialize(storage_span, flags, state_resolver,
+ allocator, &stack);
}
*out_stack = stack;
@@ -273,6 +278,11 @@
IREE_TRACE_ZONE_END(z0);
}
+IREE_API_EXPORT iree_vm_invocation_flags_t
+iree_vm_stack_invocation_flags(const iree_vm_stack_t* stack) {
+ return stack->flags;
+}
+
IREE_API_EXPORT iree_vm_stack_frame_t* iree_vm_stack_current_frame(
iree_vm_stack_t* stack) {
return stack->top ? &stack->top->frame : NULL;
@@ -501,7 +511,8 @@
iree_status_t status = iree_vm_module_resolve_source_location(
module, &frame->frame, &source_location);
if (iree_status_is_ok(status)) {
- status = iree_vm_source_location_format(&source_location, builder);
+ status = iree_vm_source_location_format(
+ &source_location, IREE_VM_SOURCE_LOCATION_FORMAT_FLAG_NONE, builder);
}
if (iree_status_is_unavailable(status)) {
// TODO(benvanik): if this is an import/export we can get that name.
diff --git a/iree/vm/stack.h b/iree/vm/stack.h
index 03bb722..f28799d 100644
--- a/iree/vm/stack.h
+++ b/iree/vm/stack.h
@@ -41,6 +41,17 @@
// The maximum size of VM stack storage; anything larger is probably a bug.
#define IREE_VM_STACK_MAX_SIZE (1 * 1024 * 1024)
+enum iree_vm_invocation_flag_bits_t {
+ IREE_VM_INVOCATION_FLAG_NONE = 0u,
+
+ // Enables tracing of execution to stderr (when available) for the invocation.
+ // See iree/base/config.h for the flags that control whether this
+ // functionality is available; specifically:
+ // -DIREE_VM_EXECUTION_TRACING_ENABLE=1
+ IREE_VM_INVOCATION_FLAG_TRACE_EXECUTION = 1u << 0,
+};
+typedef uint32_t iree_vm_invocation_flags_t;
+
typedef enum iree_vm_stack_frame_type_e {
// Represents an `[external]` frame that needs to marshal args/results.
// These frames have no source location and are tracked so that we know when
@@ -114,17 +125,19 @@
// Example:
// IREE_VM_INLINE_STACK_INITIALIZE(
// stack,
+// IREE_VM_INVOCATION_FLAG_NONE,
// iree_vm_context_state_resolver(context),
// iree_allocator_system());
// ...
// iree_vm_stack_deinitialize(stack);
-#define IREE_VM_INLINE_STACK_INITIALIZE(stack, state_resolver, allocator) \
- uint8_t __stack_storage[IREE_VM_STACK_DEFAULT_SIZE]; \
- iree_byte_span_t __stack_storage_span = \
- iree_make_byte_span(__stack_storage, sizeof(__stack_storage)); \
- iree_vm_stack_t* stack = NULL; \
- IREE_IGNORE_ERROR(iree_vm_stack_initialize( \
- __stack_storage_span, (state_resolver), (allocator), &stack));
+#define IREE_VM_INLINE_STACK_INITIALIZE(stack, flags, state_resolver, \
+ allocator) \
+ uint8_t __stack_storage[IREE_VM_STACK_DEFAULT_SIZE]; \
+ iree_byte_span_t __stack_storage_span = \
+ iree_make_byte_span(__stack_storage, sizeof(__stack_storage)); \
+ iree_vm_stack_t* stack = NULL; \
+ IREE_IGNORE_ERROR(iree_vm_stack_initialize( \
+ __stack_storage_span, (flags), (state_resolver), (allocator), &stack));
// Initializes a statically-allocated stack in |storage|.
// The contents of the |storage| can be anything upon initialization and the
@@ -146,8 +159,9 @@
// iree_vm_stack_deinitialize(stack);
// // stack_storage can now be reused/freed/etc
IREE_API_EXPORT iree_status_t iree_vm_stack_initialize(
- iree_byte_span_t storage, iree_vm_state_resolver_t state_resolver,
- iree_allocator_t allocator, iree_vm_stack_t** out_stack);
+ iree_byte_span_t storage, iree_vm_invocation_flags_t flags,
+ iree_vm_state_resolver_t state_resolver, iree_allocator_t allocator,
+ iree_vm_stack_t** out_stack);
// Deinitializes a statically-allocated |stack| previously initialized with
// iree_vm_stack_initialize.
@@ -167,13 +181,17 @@
// iree_vm_stack_allocate(..., iree_allocator_system(), &stack);
// ...
// iree_vm_stack_free(stack);
-IREE_API_EXPORT iree_status_t
-iree_vm_stack_allocate(iree_vm_state_resolver_t state_resolver,
- iree_allocator_t allocator, iree_vm_stack_t** out_stack);
+IREE_API_EXPORT iree_status_t iree_vm_stack_allocate(
+ iree_vm_invocation_flags_t flags, iree_vm_state_resolver_t state_resolver,
+ iree_allocator_t allocator, iree_vm_stack_t** out_stack);
// Frees a dynamically-allocated |stack| from iree_vm_stack_allocate.
IREE_API_EXPORT void iree_vm_stack_free(iree_vm_stack_t* stack);
+// Returns the flags controlling the invocation this stack is used with.
+IREE_API_EXPORT iree_vm_invocation_flags_t
+iree_vm_stack_invocation_flags(const iree_vm_stack_t* stack);
+
// Returns the current stack frame or nullptr if the stack is empty.
IREE_API_EXPORT iree_vm_stack_frame_t* iree_vm_stack_current_frame(
iree_vm_stack_t* stack);
diff --git a/iree/vm/stack_test.cc b/iree/vm/stack_test.cc
index 42226d6..80303df 100644
--- a/iree/vm/stack_test.cc
+++ b/iree/vm/stack_test.cc
@@ -37,8 +37,8 @@
// Tests simple stack usage, mainly just for demonstration.
TEST(VMStackTest, Usage) {
iree_vm_state_resolver_t state_resolver = {nullptr, SentinelStateResolver};
- IREE_VM_INLINE_STACK_INITIALIZE(stack, state_resolver,
- iree_allocator_system());
+ IREE_VM_INLINE_STACK_INITIALIZE(stack, IREE_VM_INVOCATION_FLAG_NONE,
+ state_resolver, iree_allocator_system());
EXPECT_EQ(nullptr, iree_vm_stack_current_frame(stack));
EXPECT_EQ(nullptr, iree_vm_stack_parent_frame(stack));
@@ -74,8 +74,8 @@
// Tests stack cleanup with unpopped frames (like during failure teardown).
TEST(VMStackTest, DeinitWithRemainingFrames) {
iree_vm_state_resolver_t state_resolver = {nullptr, SentinelStateResolver};
- IREE_VM_INLINE_STACK_INITIALIZE(stack, state_resolver,
- iree_allocator_system());
+ IREE_VM_INLINE_STACK_INITIALIZE(stack, IREE_VM_INVOCATION_FLAG_NONE,
+ state_resolver, iree_allocator_system());
iree_vm_function_t function_a = {MODULE_A_SENTINEL,
IREE_VM_FUNCTION_LINKAGE_INTERNAL, 0};
@@ -93,8 +93,8 @@
// Tests stack overflow detection.
TEST(VMStackTest, StackOverflow) {
iree_vm_state_resolver_t state_resolver = {nullptr, SentinelStateResolver};
- IREE_VM_INLINE_STACK_INITIALIZE(stack, state_resolver,
- iree_allocator_system());
+ IREE_VM_INLINE_STACK_INITIALIZE(stack, IREE_VM_INVOCATION_FLAG_NONE,
+ state_resolver, iree_allocator_system());
EXPECT_EQ(nullptr, iree_vm_stack_current_frame(stack));
EXPECT_EQ(nullptr, iree_vm_stack_parent_frame(stack));
@@ -123,8 +123,8 @@
// Tests unbalanced stack popping.
TEST(VMStackTest, UnbalancedPop) {
iree_vm_state_resolver_t state_resolver = {nullptr, SentinelStateResolver};
- IREE_VM_INLINE_STACK_INITIALIZE(stack, state_resolver,
- iree_allocator_system());
+ IREE_VM_INLINE_STACK_INITIALIZE(stack, IREE_VM_INVOCATION_FLAG_NONE,
+ state_resolver, iree_allocator_system());
iree_status_t status = iree_vm_stack_function_leave(stack);
IREE_EXPECT_STATUS_IS(IREE_STATUS_FAILED_PRECONDITION, status);
@@ -136,8 +136,8 @@
// Tests module state reuse and querying.
TEST(VMStackTest, ModuleStateQueries) {
iree_vm_state_resolver_t state_resolver = {nullptr, SentinelStateResolver};
- IREE_VM_INLINE_STACK_INITIALIZE(stack, state_resolver,
- iree_allocator_system());
+ IREE_VM_INLINE_STACK_INITIALIZE(stack, IREE_VM_INVOCATION_FLAG_NONE,
+ state_resolver, iree_allocator_system());
EXPECT_EQ(nullptr, iree_vm_stack_current_frame(stack));
EXPECT_EQ(nullptr, iree_vm_stack_parent_frame(stack));
@@ -185,8 +185,8 @@
// NOTE: always failing.
return iree_make_status(IREE_STATUS_INTERNAL);
}};
- IREE_VM_INLINE_STACK_INITIALIZE(stack, state_resolver,
- iree_allocator_system());
+ IREE_VM_INLINE_STACK_INITIALIZE(stack, IREE_VM_INVOCATION_FLAG_NONE,
+ state_resolver, iree_allocator_system());
// Push should fail if we can't query state, status should propagate.
iree_vm_function_t function_a = {MODULE_A_SENTINEL,
diff --git a/iree/vm/test/BUILD b/iree/vm/test/BUILD
index 23708eb..bc4a670 100644
--- a/iree/vm/test/BUILD
+++ b/iree/vm/test/BUILD
@@ -45,6 +45,7 @@
":global_ops_f32.vmfb",
":global_ops_i64.vmfb",
":list_ops.vmfb",
+ ":list_ops_i64.vmfb",
":list_variant_ops.vmfb",
":ref_ops.vmfb",
":shift_ops.vmfb",
@@ -170,6 +171,12 @@
)
iree_bytecode_module(
+ name = "list_ops_i64",
+ src = "list_ops_i64.mlir",
+ flags = ["-iree-vm-ir-to-bytecode-module"],
+)
+
+iree_bytecode_module(
name = "list_variant_ops",
src = "list_variant_ops.mlir",
flags = ["-iree-vm-ir-to-bytecode-module"],
diff --git a/iree/vm/test/CMakeLists.txt b/iree/vm/test/CMakeLists.txt
index e239f0f..494b6ef 100644
--- a/iree/vm/test/CMakeLists.txt
+++ b/iree/vm/test/CMakeLists.txt
@@ -37,6 +37,7 @@
"global_ops_f32.vmfb"
"global_ops_i64.vmfb"
"list_ops.vmfb"
+ "list_ops_i64.vmfb"
"list_variant_ops.vmfb"
"ref_ops.vmfb"
"shift_ops.vmfb"
@@ -241,6 +242,16 @@
iree_bytecode_module(
NAME
+ list_ops_i64
+ SRC
+ "list_ops_i64.mlir"
+ FLAGS
+ "-iree-vm-ir-to-bytecode-module"
+ PUBLIC
+)
+
+iree_bytecode_module(
+ NAME
list_variant_ops
SRC
"list_variant_ops.mlir"
diff --git a/iree/vm/test/conversion_ops.mlir b/iree/vm/test/conversion_ops.mlir
index cd35cbe..3cf741e 100644
--- a/iree/vm/test/conversion_ops.mlir
+++ b/iree/vm/test/conversion_ops.mlir
@@ -24,26 +24,4 @@
vm.return
}
- // 5.5 f32 (0x40b00000 hex) -> 1085276160 int32
- vm.export @test_bitcast_i32_f32
- vm.func @test_bitcast_i32_f32() {
- %c1 = vm.const.i32 1085276160 : i32
- %c1dno = util.do_not_optimize(%c1) : i32
- %v = vm.bitcast.i32.f32 %c1dno : i32 -> f32
- %c2 = vm.const.f32 5.5 : f32
- vm.check.eq %v, %c2, "bitcast i32 to f32" : f32
- vm.return
- }
-
- // 1085276160 int32 (0x40b00000 hex) -> 5.5 f32
- vm.export @test_bitcast_f32_i32
- vm.func @test_bitcast_f32_i32() {
- %c1 = vm.const.f32 5.5 : f32
- %c1dno = util.do_not_optimize(%c1) : f32
- %v = vm.bitcast.f32.i32 %c1dno : f32 -> i32
- %c2 = vm.const.i32 1085276160 : i32
- vm.check.eq %v, %c2, "bitcast f32 to i32" : i32
- vm.return
- }
-
}
diff --git a/iree/vm/test/conversion_ops_f32.mlir b/iree/vm/test/conversion_ops_f32.mlir
index 93fbb91..b91c652 100644
--- a/iree/vm/test/conversion_ops_f32.mlir
+++ b/iree/vm/test/conversion_ops_f32.mlir
@@ -4,6 +4,28 @@
// Casting and type conversion/emulation
//===----------------------------------------------------------------------===//
+ // 5.5 f32 (0x40b00000 hex) -> 1085276160 int32
+ vm.export @test_bitcast_i32_f32
+ vm.func @test_bitcast_i32_f32() {
+ %c1 = vm.const.i32 1085276160 : i32
+ %c1dno = util.do_not_optimize(%c1) : i32
+ %v = vm.bitcast.i32.f32 %c1dno : i32 -> f32
+ %c2 = vm.const.f32 5.5 : f32
+ vm.check.eq %v, %c2, "bitcast i32 to f32" : f32
+ vm.return
+ }
+
+ // 1085276160 int32 (0x40b00000 hex) -> 5.5 f32
+ vm.export @test_bitcast_f32_i32
+ vm.func @test_bitcast_f32_i32() {
+ %c1 = vm.const.f32 5.5 : f32
+ %c1dno = util.do_not_optimize(%c1) : f32
+ %v = vm.bitcast.f32.i32 %c1dno : f32 -> i32
+ %c2 = vm.const.i32 1085276160 : i32
+ vm.check.eq %v, %c2, "bitcast f32 to i32" : i32
+ vm.return
+ }
+
vm.export @test_cast_si32_f32_int_max
vm.func @test_cast_si32_f32_int_max() {
%c1 = vm.const.i32 2147483647 : i32
diff --git a/iree/vm/test/emitc/module_test.cc b/iree/vm/test/emitc/module_test.cc
index 4a5f8b7..1551020 100644
--- a/iree/vm/test/emitc/module_test.cc
+++ b/iree/vm/test/emitc/module_test.cc
@@ -119,8 +119,8 @@
std::vector<iree_vm_module_t*> modules = {module_};
IREE_CHECK_OK(iree_vm_context_create_with_modules(
- instance_, modules.data(), modules.size(), iree_allocator_system(),
- &context_));
+ instance_, IREE_VM_CONTEXT_FLAG_NONE, modules.data(), modules.size(),
+ iree_allocator_system(), &context_));
iree_vm_module_release(module_);
}
@@ -138,7 +138,7 @@
iree_string_view_t{qualified_name.data(), qualified_name.size()},
&function));
- return iree_vm_invoke(context_, function,
+ return iree_vm_invoke(context_, function, IREE_VM_INVOCATION_FLAG_NONE,
/*policy=*/nullptr, /*inputs=*/nullptr,
/*outputs=*/nullptr, iree_allocator_system());
}
diff --git a/iree/vm/test/list_ops.mlir b/iree/vm/test/list_ops.mlir
index 098e24c..55dbfd9 100644
--- a/iree/vm/test/list_ops.mlir
+++ b/iree/vm/test/list_ops.mlir
@@ -53,24 +53,6 @@
}
//===--------------------------------------------------------------------===//
- // vm.list.* with I64 types
- //===--------------------------------------------------------------------===//
-
- vm.export @test_i64
- vm.func @test_i64() {
- %capacity = vm.const.i32 42 : i32
- %index = vm.const.i32 41 : i32
- %max_int_plus_1 = vm.const.i64 2147483648 : i64
- %list = vm.list.alloc %capacity : (i32) -> !vm.list<i64>
- %sz = vm.list.size %list : (!vm.list<i64>) -> i32
- vm.list.resize %list, %capacity : (!vm.list<i64>, i32)
- vm.list.set.i64 %list, %index, %max_int_plus_1 : (!vm.list<i64>, i32, i64)
- %v = vm.list.get.i64 %list, %index : (!vm.list<i64>, i32) -> i64
- vm.check.eq %v, %max_int_plus_1, "list<i64>.empty.set(41, MAX_INT_PLUS_1).get(41)=MAX_INT_PLUS_1" : i64
- vm.return
- }
-
- //===--------------------------------------------------------------------===//
// vm.list.* with ref types
//===--------------------------------------------------------------------===//
diff --git a/iree/vm/test/list_ops_i64.mlir b/iree/vm/test/list_ops_i64.mlir
new file mode 100644
index 0000000..5ca41f2
--- /dev/null
+++ b/iree/vm/test/list_ops_i64.mlir
@@ -0,0 +1,21 @@
+vm.module @list_ops_i64 {
+
+ //===--------------------------------------------------------------------===//
+ // vm.list.* with I64 types
+ //===--------------------------------------------------------------------===//
+
+ vm.export @test_i64
+ vm.func @test_i64() {
+ %capacity = vm.const.i32 42 : i32
+ %index = vm.const.i32 41 : i32
+ %max_int_plus_1 = vm.const.i64 2147483648 : i64
+ %list = vm.list.alloc %capacity : (i32) -> !vm.list<i64>
+ %sz = vm.list.size %list : (!vm.list<i64>) -> i32
+ vm.list.resize %list, %capacity : (!vm.list<i64>, i32)
+ vm.list.set.i64 %list, %index, %max_int_plus_1 : (!vm.list<i64>, i32, i64)
+ %v = vm.list.get.i64 %list, %index : (!vm.list<i64>, i32) -> i64
+ vm.check.eq %v, %max_int_plus_1, "list<i64>.empty.set(41, MAX_INT_PLUS_1).get(41)=MAX_INT_PLUS_1" : i64
+ vm.return
+ }
+
+}
