Adding a minimal flag parser. It does most of what we want without any C++ in 3 modes: - entirely disabled (turned into some static variables) - argc/argv parser only for main() - flagfile parsing that uses fopen/fread It also has the nifty property of a round-tripping mode that lets you get all the current flag values and pipe them to a file to then load back in a flagfile, which should make it easier to reproduce flag-dependent configurations. Likely lots of little tweaks remaining (there always are), but this may be enough to start replacing our existing usage of the abseil flags. One thing mostly implemented is callbacks so that we can support lists (multiple instances of a flag on the command line) and actions, which aren't great in abseil flags. Fixes #3814.
diff --git a/build_tools/cmake/iree_lit_test.cmake b/build_tools/cmake/iree_lit_test.cmake index 6d18823..9d18495 100644 --- a/build_tools/cmake/iree_lit_test.cmake +++ b/build_tools/cmake/iree_lit_test.cmake
@@ -54,11 +54,13 @@ return() endif() + iree_package_ns(_PACKAGE_NS) iree_package_name(_PACKAGE_NAME) set(_NAME "${_PACKAGE_NAME}_${_RULE_NAME}") get_filename_component(_TEST_FILE_PATH ${_RULE_TEST_FILE} ABSOLUTE) + list(TRANSFORM _RULE_DATA REPLACE "^::" "${_PACKAGE_NS}::") set(_DATA_DEP_PATHS) foreach(_DATA_DEP ${_RULE_DATA}) string(REPLACE "::" "_" _DATA_DEP_NAME ${_DATA_DEP})
diff --git a/iree/base/internal/BUILD b/iree/base/internal/BUILD index 0906894..31833b4 100644 --- a/iree/base/internal/BUILD +++ b/iree/base/internal/BUILD
@@ -12,7 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Implementations for iree/base/ +# Implementations for iree/base/. +# These are not part of the IREE API. Though they may be used by external +# projects their API may change at any time. + +load("//iree:lit_test.bzl", "iree_lit_test_suite") package( default_visibility = ["//visibility:public"], @@ -151,14 +155,34 @@ cc_library( name = "flags", - srcs = ["flags.cc"], + srcs = ["flags.c"], hdrs = ["flags.h"], deps = [ + ":internal", "//iree/base:api", - "@com_google_absl//absl/flags:parse", + "//iree/base:tracing", ], ) +cc_binary( + name = "flags_demo", + srcs = ["flags_demo.c"], + deps = [ + ":flags", + "//iree/base:core_headers", + ], +) + +iree_lit_test_suite( + name = "flags_test", + srcs = ["flags_test.txt"], + data = [ + ":flags_demo", + "//iree/tools:IreeFileCheck", + ], + tags = ["hostonly"], +) + cc_library( name = "main", srcs = [
diff --git a/iree/base/internal/CMakeLists.txt b/iree/base/internal/CMakeLists.txt index 2ab0ae8..603a4bc 100644 --- a/iree/base/internal/CMakeLists.txt +++ b/iree/base/internal/CMakeLists.txt
@@ -147,13 +147,36 @@ HDRS "flags.h" SRCS - "flags.cc" + "flags.c" DEPS - absl::flags_parse + ::internal iree::base::api + iree::base::tracing PUBLIC ) +iree_cc_binary( + NAME + flags_demo + SRCS + "flags_demo.c" + DEPS + ::flags + iree::base::core_headers +) + +iree_lit_test_suite( + NAME + flags_test + SRCS + "flags_test.txt" + DATA + ::flags_demo + iree::tools::IreeFileCheck + LABELS + "hostonly" +) + iree_cc_library( NAME main
diff --git a/iree/base/internal/debugging.h b/iree/base/internal/debugging.h index 05f0594..e05773c 100644 --- a/iree/base/internal/debugging.h +++ b/iree/base/internal/debugging.h
@@ -87,6 +87,7 @@ #if defined(IREE_SANITIZER_ADDRESS) #include <sanitizer/asan_interface.h> +#include <sanitizer/lsan_interface.h> #endif // IREE_SANITIZER_ADDRESS // For whenever we want to provide specialized msan/tsan hooks:
diff --git a/iree/base/internal/flags.c b/iree/base/internal/flags.c new file mode 100644 index 0000000..f9ca811 --- /dev/null +++ b/iree/base/internal/flags.c
@@ -0,0 +1,599 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "iree/base/internal/flags.h" + +#include <errno.h> +#include <inttypes.h> +#include <limits.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "iree/base/internal/debugging.h" +#include "iree/base/tracing.h" + +#if IREE_FLAGS_ENABLE_CLI == 1 + +//===----------------------------------------------------------------------===// +// Flag manipulation utilities +//===----------------------------------------------------------------------===// + +static iree_status_t iree_flags_leaky_alloc(void* self, + iree_allocation_mode_t mode, + iree_host_size_t byte_length, + void** out_ptr) { + IREE_LEAK_CHECK_DISABLE_PUSH(); + void* ptr = malloc(byte_length); + IREE_LEAK_CHECK_DISABLE_POP(); + memset(ptr, 0, byte_length); + *out_ptr = ptr; + return iree_ok_status(); +} + +static void iree_flags_leaky_free(void* self, void* ptr) { free(ptr); } + +// Allocates heap memory that is leaked without triggering leak checkers. +// We do this so that we have valid memory for the lifetime of the process. +// The memory may still be freed but if not will not hurt anything (besides the +// private working set size). +static iree_allocator_t iree_flags_leaky_allocator() { + iree_allocator_t allocator = { + .alloc = iree_flags_leaky_alloc, + .free = iree_flags_leaky_free, + .self = NULL, + }; + return allocator; +} + +//===----------------------------------------------------------------------===// +// Flag registry +//===----------------------------------------------------------------------===// + +// Storage for registered flags. +typedef struct { + // __FILE__ of flag definition. + const char* file; + // __LINE__ of flag definition. + int line; + // Defines what data is at |storage| and how to parse/print it. + iree_flag_type_t type; + // Registered callback to issue when the flag is parsed, if any. + iree_flag_parse_callback_fn_t parse_callback; + // Registered callback to issue when the flag is to be printed, if any. + iree_flag_print_callback_fn_t print_callback; + // Direct reference to the variable storing the flag value of |type|. + void* storage; + // Name of the flag on the command line ('foo' => '--foo=value'). + iree_string_view_t name; + // Short description string. + iree_string_view_t description; +} iree_flag_t; + +// State used for flag registration and reflection. +typedef struct { + const char* program_name; + const char* usage; + + // Total number of entries in the |flags| list. + int flag_count; + // All registered flags in the executable in an undefined order. + iree_flag_t flags[IREE_FLAGS_CAPACITY]; +} iree_flag_registry_t; + +// Global flags state. +// This will persist for the lifetime of the program so that flags can be +// reparsed/dumped. If you're concerned about the .data overhead then you +// probably just want to disable the CLI support for flags entirely. +static iree_flag_registry_t iree_flag_registry = { + .program_name = NULL, + .usage = NULL, + .flag_count = 0, +}; + +int iree_flag_register(const char* file, int line, iree_flag_type_t type, + void* storage, + iree_flag_parse_callback_fn_t parse_callback, + iree_flag_print_callback_fn_t print_callback, + iree_string_view_t name, + iree_string_view_t description) { + // TODO(benvanik): make the registry a linked list and externalize the + // flag storage - then no need for a fixed count. If you're hitting this then + // file an issue :) + iree_flag_registry_t* registry = &iree_flag_registry; + IREE_ASSERT_LE(registry->flag_count + 1, IREE_FLAGS_CAPACITY, + "flag registry overflow; too many flags registered"); + int flag_ordinal = registry->flag_count++; + iree_flag_t* flag = ®istry->flags[flag_ordinal]; + flag->file = file; + flag->line = line; + flag->type = type; + flag->parse_callback = parse_callback; + flag->print_callback = print_callback; + flag->storage = storage; + flag->name = name; + flag->description = description; + return flag_ordinal; +} + +// Returns the flag registration with the given |name| or NULL if not found. +static iree_flag_t* iree_flag_lookup(iree_string_view_t name) { + iree_flag_registry_t* registry = &iree_flag_registry; + for (int i = 0; i < registry->flag_count; ++i) { + iree_flag_t* flag = ®istry->flags[i]; + if (iree_string_view_equal(flag->name, name)) { + return flag; + } + } + return NULL; +} + +static int iree_flag_cmp(const void* lhs_ptr, const void* rhs_ptr) { + const iree_flag_t* lhs = (const iree_flag_t*)lhs_ptr; + const iree_flag_t* rhs = (const iree_flag_t*)rhs_ptr; + int ret = strcmp(lhs->file, rhs->file); + if (ret == 0) { + return lhs->line - rhs->line; + } + return ret; +} + +// Sorts the flags in the flag registry by file > line. +static void iree_flag_registry_sort(iree_flag_registry_t* registry) { + qsort(registry->flags, registry->flag_count, sizeof(iree_flag_t), + iree_flag_cmp); +} + +//===----------------------------------------------------------------------===// +// Flag parsing/printing +//===----------------------------------------------------------------------===// + +void iree_flags_set_usage(const char* program_name, const char* usage) { + iree_flag_registry_t* registry = &iree_flag_registry; + registry->program_name = program_name; + registry->usage = usage; +} + +// Parses a flag value from the given string and stores it. +static iree_status_t iree_flag_parse(iree_flag_t* flag, + iree_string_view_t value) { + IREE_TRACE_ZONE_BEGIN(z0); + IREE_TRACE_ZONE_APPEND_TEXT(z0, flag->name.data, flag->name.size); + IREE_TRACE_ZONE_APPEND_TEXT(z0, value.data, value.size); + + // Insert NUL on the flag value. This is safe as the value is either coming + // from C argv memory which is mutable or a flagfile that we loaded into + // memory ourselves. + char* str_value = (char*)value.data; + if (value.size > 0) { + str_value[value.size] = 0; + } + + iree_status_t status = iree_ok_status(); + switch (flag->type) { + case IREE_FLAG_TYPE_callback: + status = flag->parse_callback(flag->name, flag->storage, value); + break; + case IREE_FLAG_TYPE_bool: + if (value.size == 0 || strcmp(str_value, "true") == 0 || + strcmp(str_value, "1") == 0) { + *(bool*)flag->storage = true; + } else { + *(bool*)flag->storage = false; + } + break; + case IREE_FLAG_TYPE_int32_t: + *(int32_t*)flag->storage = value.size ? atoi(str_value) : 0; + break; + case IREE_FLAG_TYPE_int64_t: + *(int64_t*)flag->storage = value.size ? atoll(str_value) : 0; + break; + case IREE_FLAG_TYPE_float: + *(float*)flag->storage = value.size ? (float)atof(str_value) : 0.0f; + break; + case IREE_FLAG_TYPE_double: + *(double*)flag->storage = value.size ? atof(str_value) : 0.0; + break; + case IREE_FLAG_TYPE_string: { + iree_host_size_t str_length = value.size; + if (str_length > 2) { + // Strip double quotes: "foo" -> foo. + // This may not be worth the complexity. + if (str_value[0] == '"' && str_value[str_length - 1] == '"') { + str_value[str_length - 1] = 0; + ++str_value; + str_length = str_length - 2; + } + } + *(const char**)flag->storage = str_value; + break; + } + default: + status = iree_make_status(IREE_STATUS_FAILED_PRECONDITION, + "invalid flag type %u", flag->type); + break; + } + IREE_TRACE_ZONE_END(z0); + return status; +} + +// Prints a flag value to |file| (like 'true' or '5.43'). +static void iree_flag_print(FILE* file, iree_flag_t* flag) { + if (flag->type == IREE_FLAG_TYPE_callback) { + flag->print_callback(flag->name, flag->storage, file); + return; + } + fprintf(file, "--%.*s", (int)flag->name.size, flag->name.data); + if (flag->storage == NULL) return; + switch (flag->type) { + case IREE_FLAG_TYPE_bool: + fprintf(file, "=%s", (*(bool*)flag->storage) ? "true" : "false"); + break; + case IREE_FLAG_TYPE_int32_t: + fprintf(file, "=%" PRId32, *(int32_t*)flag->storage); + break; + case IREE_FLAG_TYPE_int64_t: + fprintf(file, "=%" PRId64, *(int64_t*)flag->storage); + break; + case IREE_FLAG_TYPE_float: + fprintf(file, "=%g", *(float*)flag->storage); + break; + case IREE_FLAG_TYPE_double: + fprintf(file, "=%g", *(double*)flag->storage); + break; + case IREE_FLAG_TYPE_string: + fprintf(file, "=\"%s\"", *(const char**)flag->storage); + break; + default: + fprintf(file, "=<INVALID>"); + break; + } + fprintf(file, "\n"); +} + +// Dumps a flag definition and value to |file|. +static void iree_flag_dump(iree_flag_dump_mode_t mode, FILE* file, + iree_flag_t* flag) { + if (iree_all_bits_set(mode, IREE_FLAG_DUMP_MODE_VERBOSE)) { + if (!iree_string_view_is_empty(flag->description)) { + iree_string_view_t description = flag->description; + while (!iree_string_view_is_empty(description)) { + iree_string_view_t line; + iree_string_view_split(description, '\n', &line, &description); + if (!iree_string_view_is_empty(line)) { + fprintf(file, "# %.*s\n", (int)line.size, line.data); + } + } + } + } + iree_flag_print(file, flag); +} + +static iree_status_t iree_flags_parse_help(iree_string_view_t flag_name, + void* storage, + iree_string_view_t value) { + iree_flag_registry_t* registry = &iree_flag_registry; + + fprintf(stdout, + "# " + "====================================================================" + "========\n"); + fprintf(stdout, "# 👻 IREE: %s\n", + registry->program_name ? registry->program_name : ""); + fprintf(stdout, + "# " + "====================================================================" + "========\n\n"); + if (registry->usage) { + fprintf(stdout, "%s\n", registry->usage); + } + iree_flags_dump(IREE_FLAG_DUMP_MODE_VERBOSE, stdout); + fprintf(stdout, "\n"); + exit(0); + + return iree_ok_status(); +} +static void iree_flags_print_help(iree_string_view_t flag_name, void* storage, + FILE* file) { + fprintf(file, "# --%.*s\n", (int)flag_name.size, flag_name.data); +} +IREE_FLAG_CALLBACK(iree_flags_parse_help, iree_flags_print_help, NULL, help, + "Displays command line usage information."); + +// Removes argument |arg| from the argument list. +static void iree_flags_remove_arg(int arg, int* argc_ptr, char*** argv_ptr) { + int argc = *argc_ptr; + char** argv = *argv_ptr; + memmove(&argv[arg], &argv[arg + 1], (argc - arg) * sizeof(char*)); + *argc_ptr = argc - 1; +} + +iree_status_t iree_flags_parse(iree_flags_parse_mode_t mode, int* argc_ptr, + char*** argv_ptr) { + if (argc_ptr == NULL || argv_ptr == NULL || *argc_ptr == 0) { + // No flags; that's fine - in some environments flags aren't supported. + return iree_ok_status(); + } + + // Always sort the registry; though we may parse flags multiple times this is + // not a hot path and this is easier than trying to keep track of whether we + // need to or not. + iree_flag_registry_sort(&iree_flag_registry); + + int argc = *argc_ptr; + char** argv = *argv_ptr; + + for (int arg_ordinal = 1; arg_ordinal < argc; ++arg_ordinal) { + iree_string_view_t arg = iree_make_cstring_view(argv[arg_ordinal]); + + // Strip whitespace. + arg = iree_string_view_trim(arg); + + // Position arguments are ignored; they may appear anywhere in the list. + if (!iree_string_view_starts_with(arg, iree_make_cstring_view("--"))) { + continue; + } + + // Strip `--`. + arg = iree_string_view_remove_prefix(arg, 2); + + // Split into `flag_name` = `flag_value`. + iree_string_view_t flag_name; + iree_string_view_t flag_value; + iree_string_view_split(arg, '=', &flag_name, &flag_value); + flag_name = iree_string_view_trim(flag_name); + flag_value = iree_string_view_trim(flag_value); + + // Lookup the flag by name. + iree_flag_t* flag = iree_flag_lookup(flag_name); + if (!flag) { + // If --undefok allows undefined flags then we just skip this one. Note + // that we leave it in the argument list so that subsequent flag parsers + // can try to handle it. + if (iree_all_bits_set(mode, IREE_FLAGS_PARSE_MODE_UNDEFINED_OK)) { + continue; + } + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "flag '%.*s' not recognized", (int)flag_name.size, + flag_name.data); + } + + // Parse and store the flag value. + IREE_RETURN_IF_ERROR(iree_flag_parse(flag, flag_value)); + + // Splice out the flag from the argv list. + iree_flags_remove_arg(arg_ordinal, &argc, &argv); + --arg_ordinal; + } + + *argc_ptr = argc; + return iree_ok_status(); +} + +void iree_flags_parse_checked(iree_flags_parse_mode_t mode, int* argc, + char*** argv) { + IREE_TRACE_ZONE_BEGIN(z0); + for (int i = 0; i < *argc; ++i) { + IREE_TRACE_ZONE_APPEND_TEXT_CSTRING(z0, (*argv)[i]); + } + iree_status_t status = iree_flags_parse(mode, argc, argv); + IREE_TRACE_ZONE_END(z0); + if (iree_status_is_ok(status)) return; + + fprintf(stderr, "\x1b[31mFLAGS ERROR: (╯°â–¡°)╯︵👻\x1b[0m\n"); + char* buffer = NULL; + iree_host_size_t buffer_length = 0; + iree_status_to_string(status, &buffer, &buffer_length); + fprintf(stderr, "%.*s\n\n", (int)buffer_length, buffer); + fflush(stderr); + iree_allocator_free(iree_allocator_system(), buffer); + + exit(EXIT_FAILURE); +} + +void iree_flags_dump(iree_flag_dump_mode_t mode, FILE* file) { + IREE_TRACE_ZONE_BEGIN(z0); + + // Always sort the registry; though we may dump flags multiple times this is + // not a hot path and this is easier than trying to keep track of whether we + // need to or not. + iree_flag_registry_sort(&iree_flag_registry); + + const char* last_file = NULL; + for (size_t i = 0; i < iree_flag_registry.flag_count; ++i) { + iree_flag_t* flag = &iree_flag_registry.flags[i]; + if (iree_all_bits_set(mode, IREE_FLAG_DUMP_MODE_VERBOSE)) { + if (last_file) { + fprintf(file, "\n"); + } + if (!last_file || strcmp(flag->file, last_file) != 0) { + fprintf(file, + "# " + "===-----------------------------------------------------------" + "-----------===\n"); + fprintf(file, "# Flags in %s:%d\n", flag->file, flag->line); + fprintf(file, + "# " + "===-----------------------------------------------------------" + "-----------===\n\n"); + last_file = flag->file; + } + } + iree_flag_dump(mode, file, flag); + } + + IREE_TRACE_ZONE_END(z0); +} + +//===----------------------------------------------------------------------===// +// --flagfile= support +//===----------------------------------------------------------------------===// +// NOTE: this is conditionally enabled as some platforms may not have IO. + +#if IREE_FLAGS_ENABLE_FLAG_FILE == 1 + +// TODO(benvanik): use this to replace file_io.cc. +static iree_status_t iree_file_read_contents(const char* path, + iree_allocator_t allocator, + iree_byte_span_t* out_contents) { + IREE_TRACE_ZONE_BEGIN(z0); + *out_contents = iree_make_byte_span(NULL, 0); + FILE* file = fopen(path, "rb"); + if (file == NULL) { + IREE_TRACE_ZONE_END(z0); + return iree_make_status(iree_status_code_from_errno(errno), + "failed to open file '%s'", path); + } + iree_status_t status = iree_ok_status(); + if (fseek(file, 0, SEEK_END) == -1) { + status = iree_make_status(iree_status_code_from_errno(errno), "seek (end)"); + } + size_t file_size = 0; + if (iree_status_is_ok(status)) { + file_size = ftell(file); + if (file_size == -1L) { + status = + iree_make_status(iree_status_code_from_errno(errno), "size query"); + } + } + if (iree_status_is_ok(status)) { + if (fseek(file, 0, SEEK_SET) == -1) { + status = + iree_make_status(iree_status_code_from_errno(errno), "seek (beg)"); + } + } + // Allocate +1 to force a trailing \0 in case this is a string. + char* contents = NULL; + if (iree_status_is_ok(status)) { + status = iree_allocator_malloc(allocator, file_size + 1, (void**)&contents); + } + if (iree_status_is_ok(status)) { + if (fread(contents, file_size, 1, file) != 1) { + status = + iree_make_status(iree_status_code_from_errno(errno), + "unable to read entire file contents of '%s'", path); + } + } + if (iree_status_is_ok(status)) { + contents[file_size] = 0; // NUL + *out_contents = iree_make_byte_span(contents, file_size); + } else { + iree_allocator_free(allocator, contents); + } + fclose(file); + IREE_TRACE_ZONE_END(z0); + return status; +} + +// Parses a newline-separated list of flags from a file. +static iree_status_t iree_flags_parse_file(iree_string_view_t file_path) { + // Read file contents. + // NOTE: we intentionally leak the contents here so that the flags remain in + // memory in case they are referenced. + // NOTE: safe to use file_path.data here as it will always have a NUL + // terminator. + iree_allocator_t allocator = iree_flags_leaky_allocator(); + iree_byte_span_t file_contents; + IREE_RETURN_IF_ERROR( + iree_file_read_contents(file_path.data, allocator, &file_contents), + "while trying to parse flagfile"); + + // Run through the file line-by-line. + int line_number = 0; + iree_string_view_t contents = iree_make_string_view( + (const char*)file_contents.data, file_contents.data_length); + while (!iree_string_view_is_empty(contents)) { + // Split into a single line and the entire rest of the file contents. + iree_string_view_t line; + iree_string_view_split(contents, '\n', &line, &contents); + ++line_number; + + // Strip whitespace. + line = iree_string_view_trim(line); + if (iree_string_view_is_empty(line)) continue; + + // Ignore comments. + if (iree_string_view_starts_with(line, iree_make_cstring_view("#")) || + iree_string_view_starts_with(line, iree_make_cstring_view("//"))) { + continue; + } + + // Strip `--`. + if (!iree_string_view_starts_with(line, iree_make_cstring_view("--"))) { + // Positional arguments can't be specified in flag files. + return iree_make_status( + IREE_STATUS_INVALID_ARGUMENT, + "%.*s:%d: positional arguments not allowed in flag files", + (int)file_path.size, file_path.data, line_number); + } + line = iree_string_view_remove_prefix(line, 2); + + // Split into `flag_name` = `flag_value`. + iree_string_view_t flag_name; + iree_string_view_t flag_value; + iree_string_view_split(line, '=', &flag_name, &flag_value); + flag_name = iree_string_view_trim(flag_name); + flag_value = iree_string_view_trim(flag_value); + + // Lookup the flag by name. + iree_flag_t* flag = iree_flag_lookup(flag_name); + if (!flag) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "%.*s:%d: flag '%.*s' not recognized", + (int)file_path.size, file_path.data, line_number, + (int)flag_name.size, flag_name.data); + } + + // Parse the flag value. + IREE_RETURN_IF_ERROR(iree_flag_parse(flag, flag_value), + "%.*s:%d: while parsing flag '%.*s'", + (int)file_path.size, file_path.data, line_number, + (int)line.size, line.data); + } + + // NOTE: we intentionally leak the memory as flags may continue to reference + // segments of it for their string values. + return iree_ok_status(); +} + +static iree_status_t iree_flags_parse_flagfile(iree_string_view_t flag_name, + void* storage, + iree_string_view_t value) { + if (iree_string_view_is_empty(value)) { + return iree_make_status(IREE_STATUS_INVALID_ARGUMENT, + "--%.*s= requires a file path", (int)flag_name.size, + flag_name.data); + } + + IREE_TRACE_ZONE_BEGIN(z0); + IREE_TRACE_ZONE_APPEND_TEXT(z0, value.data, value.size); + iree_status_t status = iree_flags_parse_file(value); + IREE_TRACE_ZONE_END(z0); + + return status; +} +static void iree_flags_print_flagfile(iree_string_view_t flag_name, + void* storage, FILE* file) { + fprintf(file, "# --%.*s=[path]\n", (int)flag_name.size, flag_name.data); +} +IREE_FLAG_CALLBACK(iree_flags_parse_flagfile, iree_flags_print_flagfile, NULL, + flagfile, + "Parses a newline-separated list of flags from a file.\n" + "Flags are parsed at the point where the flagfile is " + "specified\nand following flags may override the parsed " + "values."); + +#endif // IREE_FLAGS_ENABLE_FLAG_FILE + +#endif // IREE_FLAGS_ENABLE_CLI
diff --git a/iree/base/internal/flags.cc b/iree/base/internal/flags.cc deleted file mode 100644 index 8d0145a..0000000 --- a/iree/base/internal/flags.cc +++ /dev/null
@@ -1,54 +0,0 @@ -// Copyright 2019 Google LLC -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "iree/base/internal/flags.h" - -#include <stdlib.h> -#include <string.h> - -// TODO(#3814): replace abseil with pretty much anything else. -#include "absl/flags/parse.h" - -iree_status_t iree_flags_parse(int* argc, char*** argv) { - if (argc == nullptr || argv == nullptr || *argc == 0) { - // No flags; that's fine - in some environments flags aren't supported. - return iree_ok_status(); - } - - auto positional_args = absl::ParseCommandLine(*argc, *argv); - if (positional_args.size() < *argc) { - // Edit the passed argument refs to only include positional args. - *argc = static_cast<int>(positional_args.size()); - for (int i = 0; i < *argc; ++i) { - (*argv)[i] = positional_args[i]; - } - (*argv)[*argc + 1] = nullptr; - } - - return iree_ok_status(); -} - -void iree_flags_parse_checked(int* argc, char*** argv) { - iree_status_t status = iree_flags_parse(argc, argv); - if (iree_status_is_cancelled(status)) { - exit(EXIT_SUCCESS); - return; - } - if (!iree_status_is_ok(status)) { - // TODO(#2843): replace C++ logging. - iree_status_ignore(status); - exit(EXIT_FAILURE); - return; - } -}
diff --git a/iree/base/internal/flags.h b/iree/base/internal/flags.h index ef2c9da..d519f61 100644 --- a/iree/base/internal/flags.h +++ b/iree/base/internal/flags.h
@@ -15,50 +15,273 @@ #ifndef IREE_BASE_INTERNAL_FLAGS_H_ #define IREE_BASE_INTERNAL_FLAGS_H_ +#include <stdio.h> + #include "iree/base/api.h" +#include "iree/base/target_platform.h" #ifdef __cplusplus extern "C" { #endif // __cplusplus +// 1 to enable command line parsing from argc/argv; 0 otherwise. +// When parsing is disabled flags are just variables that can still be queried +// and manually overridden by code if desired. +#if !defined(IREE_FLAGS_ENABLE_CLI) +#define IREE_FLAGS_ENABLE_CLI 1 +#endif // !IREE_FLAGS_ENABLE_CLI + +// 1 to enable --flagfile= support. +#if !defined(IREE_FLAGS_ENABLE_FLAG_FILE) +#define IREE_FLAGS_ENABLE_FLAG_FILE 1 +#endif // !IREE_FLAGS_ENABLE_FLAG_FILE + +// Maximum number of flags that can be registered in a single binary. +#define IREE_FLAGS_CAPACITY 64 + +//===----------------------------------------------------------------------===// +// Static initialization utility +//===----------------------------------------------------------------------===// +// This declares a static initialization function with the given name. +// Usage: +// IREE_STATIC_INITIALIZER(initializer_name) { +// // Do something here! Note that initialization order is undefined and +// // what you do should be tolerant to that. +// +// // If you want a finalizer (you probably don't; they may not get run) +// // then you can use atexit: +// atexit(some_finalizer_fn); +// } + +#ifdef __cplusplus + +#define IREE_STATIC_INITIALIZER(f) \ + static void f(void); \ + struct f##_t_ { \ + f##_t_(void) { f(); } \ + }; \ + static f##_t_ f##_; \ + static void f(void) + +#elif defined(IREE_COMPILER_MSVC) + +// `__attribute__((constructor))`-like behavior in MSVC. See: +// https://docs.microsoft.com/en-us/cpp/c-runtime-library/crt-initialization?view=msvc-160 + +#pragma section(".CRT$XCU", read) +#define IREE_STATIC_INITIALIZER_IMPL(f, p) \ + static void f(void); \ + __declspec(allocate(".CRT$XCU")) void (*f##_)(void) = f; \ + __pragma(comment(linker, "/include:" p #f "_")) static void f(void) +#ifdef _WIN64 +#define IREE_STATIC_INITIALIZER(f) IREE_STATIC_INITIALIZER_IMPL(f, "") +#else +#define IREE_STATIC_INITIALIZER(f) IREE_STATIC_INITIALIZER_IMPL(f, "_") +#endif // _WIN64 + +#else + +#define IREE_STATIC_INITIALIZER(f) \ + static void f(void) __attribute__((constructor)); \ + static void f(void) + +#endif // __cplusplus / MSVC + +//===----------------------------------------------------------------------===// +// Flag definition +//===----------------------------------------------------------------------===// + +typedef enum { + IREE_FLAG_DUMP_MODE_DEFAULT = 0, + IREE_FLAG_DUMP_MODE_VERBOSE = 1u << 0, +} iree_flag_dump_mode_t; + +#if IREE_FLAGS_ENABLE_CLI == 1 + +// Types of flags supported by the parser. +typedef enum { + // Empty/unspecified sentinel. + IREE_FLAG_TYPE_none = 0, + // Custom parsing callback; see IREE_FLAG_CALLBACK. + IREE_FLAG_TYPE_callback = 1, + // Boolean flag: + // --foo (set true) + // --foo=true | --foo=false + IREE_FLAG_TYPE_bool, + // 32-bit integer flag: + // --foo=123 + IREE_FLAG_TYPE_int32_t, + // 64-bit integer flag: + // --foo=123 + IREE_FLAG_TYPE_int64_t, + // 32-bit floating-point flag: + // --foo=1.2 + IREE_FLAG_TYPE_float, + // 64-bit floating-point flag: + // --foo=1.2 + IREE_FLAG_TYPE_double, + // String flag: + // --foo=abc + // --foo="a b c" + // Holds a reference to constant string data; assigned values must remain + // live for as long as the flag value references them. + IREE_FLAG_TYPE_string, +} iree_flag_type_t; + +#define IREE_FLAG_CTYPE_bool bool +#define IREE_FLAG_CTYPE_int32_t int32_t +#define IREE_FLAG_CTYPE_int64_t int64_t +#define IREE_FLAG_CTYPE_float float +#define IREE_FLAG_CTYPE_double double +#define IREE_FLAG_CTYPE_string const char* + +// Custom callback issued for each time the flag is seen during parsing. +// The |value| provided will already be trimmed and may be empty. For +// compatibility with non-IREE APIs there will be a NUL terminator immediately +// following the flag value in memory such that `value.data` can be used as a +// C-string. +typedef iree_status_t(IREE_API_PTR* iree_flag_parse_callback_fn_t)( + iree_string_view_t flag_name, void* storage, iree_string_view_t value); + +// Custom callback issued for each time the flag is to be printed. +// The callback should print the flag and its value to |file|. +// Example: `--my_flag=value\n` +typedef void(IREE_API_PTR* iree_flag_print_callback_fn_t)( + iree_string_view_t flag_name, void* storage, FILE* file); + +int iree_flag_register(const char* file, int line, iree_flag_type_t type, + void* storage, + iree_flag_parse_callback_fn_t parse_callback, + iree_flag_print_callback_fn_t print_callback, + iree_string_view_t name, iree_string_view_t description); + +// Defines a flag with the given |type| and |name|. +// +// Conceptually the flag is just a variable and can be loaded/stored: +// IREE_FLAG(bool, foo, true, "hello"); +// => +// static bool FLAG_foo = true; +// ... +// if (FLAG_foo) do_something(); +// +// If flag parsing is enabled with IREE_FLAGS_ENABLE_CLI == 1 then the flag +// value can be specified on the command line with --name: +// --foo +// --foo=true +// +// See iree_flag_type_t for the types supported and how they are parsed. +#define IREE_FLAG(type, name, default_value, description) \ + static IREE_FLAG_CTYPE_##type FLAG_##name = (default_value); \ + IREE_STATIC_INITIALIZER(iree_flag_register_##name) { \ + iree_flag_register(__FILE__, __LINE__, IREE_FLAG_TYPE_##type, \ + (void**)&(FLAG_##name), /*parse_callback=*/NULL, \ + /*print_callback=*/NULL, iree_make_cstring_view(#name), \ + iree_make_cstring_view(description)); \ + } + +// Defines a flag issues |callback| for custom parsing. +// +// Usage: +// iree_status_t parse_callback(const char* flag_name, void* storage, +// iree_string_view_t value) { +// // Parse |value| and store in |storage|, however you want. +// // Returning IREE_STATUS_INVALID_ARGUMENT will trigger --help. +// int* storage_ptr = (int*)storage; +// printf("hello! %d", (*storage_ptr)++); +// return iree_ok_status(); +// } +// void print_callback(const char* flag_name, void* storage, FILE* file) { +// // Print the value in |storage|, however you want. For repeated fields +// // you can print multiple separated by newlines. +// int* storage_ptr = (int*)storage; +// fprintf(file, "--say_hello=%d\n", *storage_ptr); +// } +// int my_storage = 0; +// IREE_FLAG_CALLBACK(parse_callback, print_callback, &my_storage, +// say_hello, "Say hello!"); +#define IREE_FLAG_CALLBACK(parse_callback, print_callback, storage, name, \ + description) \ + IREE_STATIC_INITIALIZER(iree_flag_register_##name) { \ + iree_flag_register(__FILE__, __LINE__, IREE_FLAG_TYPE_callback, \ + (void*)storage, parse_callback, print_callback, \ + iree_make_cstring_view(#name), \ + iree_make_cstring_view(description)); \ + } + +#else + +#define IREE_FLAG(type, name, default_value, description) \ + static IREE_FLAG_CTYPE_##type name = (default_value); + +#define IREE_FLAG_CALLBACK(parse_callback, print_callback, storage, name, \ + description) + +#endif // IREE_FLAGS_ENABLE_CLI + //===----------------------------------------------------------------------===// // Flag parsing //===----------------------------------------------------------------------===// +// Controls how flag parsing is performed. +typedef enum { + IREE_FLAGS_PARSE_MODE_DEFAULT = 0, + // Do not error out on undefined flags; leave them in the list. + // Useful when needing to chain multiple flag parsers together. + IREE_FLAGS_PARSE_MODE_UNDEFINED_OK = 1u << 0, +} iree_flags_parse_mode_t; + +#if IREE_FLAGS_ENABLE_CLI == 1 + +// Sets the usage information printed when --help is passed on the command line. +// Both strings must remain live for the lifetime of the program. +void iree_flags_set_usage(const char* program_name, const char* usage); + // Parses flags from the given command line arguments. // All flag-style arguments ('--foo', '-f', etc) will be consumed and argc/argv // will be updated to contain only the program name (index 0) and any remaining // positional arguments. // -// Returns success if all flags were parsed and execution should continue. -// May return IREE_STATUS_CANCELLED if execution should be cancelled gracefully -// such as when --help is used. +// Returns 0 if all flags were parsed and execution should continue. +// Returns >0 if execution should be cancelled such as when --help is used. +// Returns <0 if parsing fails. // // Usage: // extern "C" int main(int argc, char** argv) { // iree_status_t status = iree_flags_parse(&argc, &argv); -// if (iree_status_is_cancelled(status)) return 0; -// if (!iree_status_is_ok(status)) { -// // TODO(#2843): replace C++ logging. -// LOG(ERROR) << status; -// iree_status_ignore(status); -// return 1; -// } +// if (!iree_status_is_ok(status)) { exit(1); } // consume_positional_args(argc, argv); // return 0; // } // // Example: -// argc = 4, argv = ['program', 'abc', '--flag=2', '-p'] +// argc = 4, argv = ['program', 'abc', '--flag=2'] // Results: // argc = 2, argv = ['program', 'abc'] -iree_status_t iree_flags_parse(int* argc, char*** argv); +iree_status_t iree_flags_parse(iree_flags_parse_mode_t mode, int* argc, + char*** argv); // Parses flags as with iree_flags_parse but will use exit() or abort(). // WARNING: this almost always what you want in a command line tool and *never* // what you want when embedded in a host process. You don't want to have a flag // typo and shut down your entire server/sandbox/Android app/etc. -void iree_flags_parse_checked(int* argc, char*** argv); +void iree_flags_parse_checked(iree_flags_parse_mode_t mode, int* argc, + char*** argv); + +// Dumps all flags and their current values to the given |file|. +void iree_flags_dump(iree_flag_dump_mode_t mode, FILE* file); + +#else + +inline void iree_flags_set_usage(const char* program_name, const char* usage) {} +inline int iree_flags_parse(iree_flags_parse_mode_t mode, int* argc, + char*** argv) { + return 0; +} +inline void iree_flags_parse_checked(iree_flags_parse_mode_t mode, int* argc, + char*** argv) {} +inline void iree_flags_dump(iree_flag_dump_mode_t mode, FILE* file) {} + +#endif // IREE_FLAGS_ENABLE_CLI #ifdef __cplusplus } // extern "C"
diff --git a/iree/base/internal/flags_demo.c b/iree/base/internal/flags_demo.c new file mode 100644 index 0000000..34cd671 --- /dev/null +++ b/iree/base/internal/flags_demo.c
@@ -0,0 +1,67 @@ +// Copyright 2021 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <inttypes.h> +#include <stdint.h> +#include <stdio.h> + +#include "iree/base/internal/flags.h" + +IREE_FLAG(bool, test_bool, false, "A boolean value."); +IREE_FLAG(int32_t, test_int32, 123, "An int32_t value."); +IREE_FLAG(int64_t, test_int64, 555, "An int64_t value."); +IREE_FLAG(float, test_float, 1.0f, "A float value."); +IREE_FLAG(string, test_string, "some default", "A string\nvalue."); + +static iree_status_t parse_callback(iree_string_view_t flag_name, void* storage, + iree_string_view_t value) { + int* count_ptr = (int*)storage; + if (strcmp(value.data, "FORCE_FAILURE") == 0) { + return iree_make_status(IREE_STATUS_INTERNAL, + "callbacks can do verification"); + } + *count_ptr += atoi(value.data); + return iree_ok_status(); +} +static void print_callback(iree_string_view_t flag_name, void* storage, + FILE* file) { + int* count_ptr = (int*)storage; + fprintf(file, "--%.*s=%d\n", (int)flag_name.size, flag_name.data, *count_ptr); +} +static int callback_count = 0; +IREE_FLAG_CALLBACK(parse_callback, print_callback, &callback_count, + test_callback, "Callback!"); + +int main(int argc, char** argv) { + // Parse flags, updating argc/argv with position arguments. + iree_flags_parse_checked(IREE_FLAGS_PARSE_MODE_DEFAULT, &argc, &argv); + + // Report parsed flag values: + printf("FLAG[test_bool] = %s\n", FLAG_test_bool ? "true" : "false"); + printf("FLAG[test_int32] = %" PRId32 "\n", FLAG_test_int32); + printf("FLAG[test_int64] = %" PRId64 "\n", FLAG_test_int64); + printf("FLAG[test_float] = %g\n", FLAG_test_float); + printf("FLAG[test_string] = %s\n", FLAG_test_string); + printf("FLAG[test_callback] = %d\n", callback_count); + + // Report positional arguments: + for (int i = 0; i < argc; ++i) { + printf("ARG(%d) = %s\n", i, argv[i]); + } + + // Dump all flags back out for round-tripping: + iree_flags_dump(IREE_FLAG_DUMP_MODE_DEFAULT, stdout); + + return 0; +}
diff --git a/iree/base/internal/flags_test.txt b/iree/base/internal/flags_test.txt new file mode 100644 index 0000000..8ffaa5a --- /dev/null +++ b/iree/base/internal/flags_test.txt
@@ -0,0 +1,93 @@ +// RUN: ( flags_demo ) | IreeFileCheck --check-prefix=NO-FLAGS %s +// NO-FLAGS: FLAG[test_bool] = false +// NO-FLAGS: FLAG[test_int32] = 123 +// NO-FLAGS: FLAG[test_int64] = 555 +// NO-FLAGS: FLAG[test_float] = 1 +// NO-FLAGS: FLAG[test_string] = some default +// NO-FLAGS: FLAG[test_callback] = 0 +// NO-FLAGS: ARG(0) ={{.+}}flags_demo + +// RUN: ( flags_demo --help ) | IreeFileCheck --check-prefix=FLAGS-HELP %s +// FLAGS-HELP: # {{.+}} IREE +// FLAGS-HELP: # Flags in {{.+}}flags.c +// FLAGS-HELP: # Displays command line usage information. +// FLAGS-HELP: --help +// FLAGS-HELP: # Flags in {{.+}}flags_demo.c +// FLAGS-HELP: # A boolean value. +// FLAGS-HELP: --test_bool=false +// FLAGS-HELP: # An int32_t value. +// FLAGS-HELP: --test_int32=123 +// FLAGS-HELP: # An int64_t value. +// FLAGS-HELP: --test_int64=555 +// FLAGS-HELP: # A float value. +// FLAGS-HELP: --test_float=1 +// FLAGS-HELP: # A string +// FLAGS-HELP: # value. +// FLAGS-HELP: --test_string="some default" +// FLAGS-HELP: # Callback! +// FLAGS-HELP: --test_callback=0 + +// RUN: ( flags_demo --unknown-flag 2>&1 || [[ $? == 1 ]] ) | IreeFileCheck --check-prefix=UNKNOWN-FLAG %s +// UNKNOWN-FLAG: INVALID_ARGUMENT; flag 'unknown-flag' not recognized + +// RUN: ( flags_demo --test_bool=true ) | IreeFileCheck --check-prefix=FLAG-BOOL-TRUE %s +// FLAG-BOOL-TRUE: FLAG[test_bool] = true +// RUN: ( flags_demo --test_bool=1 ) | IreeFileCheck --check-prefix=FLAG-BOOL-1 %s +// FLAG-BOOL-1: FLAG[test_bool] = true +// RUN: ( flags_demo --test_bool=true --test_bool=false ) | IreeFileCheck --check-prefix=FLAG-BOOL-OVERRIDE %s +// FLAG-BOOL-OVERRIDE: FLAG[test_bool] = false + +// RUN: ( flags_demo --test_int32=456 ) | IreeFileCheck --check-prefix=FLAG-INT32 %s +// FLAG-INT32: FLAG[test_int32] = 456 +// RUN: ( flags_demo --test_int32=-2147483648 ) | IreeFileCheck --check-prefix=FLAG-INT32-MIN %s +// FLAG-INT32-MIN: FLAG[test_int32] = -2147483648 +// RUN: ( flags_demo --test_int32=2147483647 ) | IreeFileCheck --check-prefix=FLAG-INT32-MAX %s +// FLAG-INT32-MAX: FLAG[test_int32] = 2147483647 + +// RUN: ( flags_demo --test_int64=902834 ) | IreeFileCheck --check-prefix=FLAG-INT64 %s +// FLAG-INT64: FLAG[test_int64] = 902834 +// RUN: ( flags_demo --test_int64=-9223372036854775808 ) | IreeFileCheck --check-prefix=FLAG-INT64-MIN %s +// FLAG-INT64-MIN: FLAG[test_int64] = -9223372036854775808 +// RUN: ( flags_demo --test_int64=9223372036854775807 ) | IreeFileCheck --check-prefix=FLAG-INT64-MAX %s +// FLAG-INT64-MAX: FLAG[test_int64] = 9223372036854775807 + +// RUN: ( flags_demo --test_float=1.1234 ) | IreeFileCheck --check-prefix=FLAG-FLOAT %s +// FLAG-FLOAT: FLAG[test_float] = 1.1234 + +// RUN: ( flags_demo --test_string= ) | IreeFileCheck --check-prefix=FLAG-STRING-EMPTY %s +// FLAG-STRING-EMPTY: FLAG[test_string] = +// RUN: ( flags_demo --test_string=abc ) | IreeFileCheck --check-prefix=FLAG-STRING-ABC %s +// FLAG-STRING-ABC: FLAG[test_string] = abc +// RUN: ( flags_demo --test_string="with some space" ) | IreeFileCheck --check-prefix=FLAG-STRING-SPACES %s +// FLAG-STRING-SPACES: FLAG[test_string] = with some space + +// RUN: ( flags_demo --test_callback=1 ) | IreeFileCheck --check-prefix=FLAG-CALLBACK-1 %s +// FLAG-CALLBACK-1: FLAG[test_callback] = 1 +// RUN: ( flags_demo --test_callback=4 ) | IreeFileCheck --check-prefix=FLAG-CALLBACK-4 %s +// FLAG-CALLBACK-4: FLAG[test_callback] = 4 +// RUN: ( flags_demo --test_callback=FORCE_FAILURE 2>&1 || [[ $? == 1 ]] ) | IreeFileCheck --check-prefix=FLAG-CALLBACK-ERROR %s +// FLAG-CALLBACK-ERROR: INTERNAL; callbacks can do verification + +// RUN: ( flags_demo arg1 ) | IreeFileCheck --check-prefix=FLAG-POSITIONAL-1 %s +// FLAG-POSITIONAL-1: ARG(1) = arg1 +// RUN: ( flags_demo arg1 arg2 arg3 ) | IreeFileCheck --check-prefix=FLAG-POSITIONAL-3 %s +// FLAG-POSITIONAL-3: ARG(1) = arg1 +// FLAG-POSITIONAL-3: ARG(2) = arg2 +// FLAG-POSITIONAL-3: ARG(3) = arg3 + +// RUN: ( flags_demo --test_bool=true --flagfile=not_found.txt 2>&1 || [[ $? == 1 ]] ) | IreeFileCheck --check-prefix=MISSING-FLAGFILE %s +// MISSING-FLAGFILE: NOT_FOUND; failed to open file 'not_found.txt' + +// RUN: ( flags_demo --test_bool=true --flagfile=%s ) | IreeFileCheck --check-prefix=FLAGFILE %s +# Comments are ignored. +// FLAGFILE: FLAG[test_bool] = false +--test_bool=false +// FLAGFILE: FLAG[test_int64] = 123111 + --test_int64=123111 +// FLAGFILE: FLAG[test_float] = 55.1 +--test_float=55.1 +// FLAGFILE: FLAG[test_string] = override spaces +--test_string="override spaces" + + +# NOTE: above two lines are to test that vertical whitespace is ok.
diff --git a/iree/base/tracing.h b/iree/base/tracing.h index f73ac4c..0293313 100644 --- a/iree/base/tracing.h +++ b/iree/base/tracing.h
@@ -410,6 +410,8 @@ #define IREE_TRACE_ZONE_SET_COLOR(zone_id, color_xrgb) #define IREE_TRACE_ZONE_APPEND_VALUE(zone_id, value) #define IREE_TRACE_ZONE_APPEND_TEXT(zone_id, ...) +#define IREE_TRACE_ZONE_APPEND_TEXT_CSTRING(zone_id, value) +#define IREE_TRACE_ZONE_APPEND_TEXT_STRING_VIEW(zone_id, value, value_length) #define IREE_TRACE_ZONE_END(zone_id) #define IREE_RETURN_AND_END_ZONE_IF_ERROR(zone_id, ...) \ IREE_RETURN_IF_ERROR(__VA_ARGS__)