pw_tokenizer: Replace string literals with tokens
pw_tokenizer provides macros that replace printf-style string literals
with 32-bit hashes at compile time. The string literals are removed
from the resulting binary, which dramatically reduces the binary size.
Like any printf-style string, binary versions of the strings can be
formatted with arguments and then transmitted or stored.
The pw_tokenizer module is general purpose, but its most common use case
is binary logging. In binary logging, human-readable text logs are
replaced with binary tokens. These are decoded off-device.
This commit includes the C and C++ code for tokenizing strings. It also
includes a C++ library for decoding tokenized strings.
Change-Id: I6d5737ab2d6dfdd76dcf70c852b547fdcd68d683
diff --git a/BUILD.gn b/BUILD.gn
index c30696b..b7acdea 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -55,6 +55,7 @@
"$dir_pw_span",
"$dir_pw_status",
"$dir_pw_string",
+ "$dir_pw_tokenizer",
"$dir_pw_unit_test",
"$dir_pw_varint",
]
@@ -69,6 +70,7 @@
"$dir_pw_span:tests",
"$dir_pw_status:tests",
"$dir_pw_string:tests",
+ "$dir_pw_tokenizer:tests",
"$dir_pw_unit_test:tests",
"$dir_pw_varint:tests",
]
diff --git a/docs/BUILD.gn b/docs/BUILD.gn
index d4c3767..050b326 100644
--- a/docs/BUILD.gn
+++ b/docs/BUILD.gn
@@ -57,5 +57,6 @@
"$dir_pw_status:docs",
"$dir_pw_string:docs",
"$dir_pw_target_runner:docs",
+ "$dir_pw_tokenizer:docs",
]
}
diff --git a/modules.gni b/modules.gni
index 59faf73..fbbee69 100644
--- a/modules.gni
+++ b/modules.gni
@@ -36,6 +36,7 @@
dir_pw_status = "$dir_pigweed/pw_status"
dir_pw_string = "$dir_pigweed/pw_string"
dir_pw_target_runner = "$dir_pigweed/pw_target_runner"
+dir_pw_tokenizer = "$dir_pigweed/pw_tokenizer"
dir_pw_toolchain = "$dir_pigweed/pw_toolchain"
dir_pw_unit_test = "$dir_pigweed/pw_unit_test"
dir_pw_varint = "$dir_pigweed/pw_varint"
diff --git a/pw_tokenizer/BUILD b/pw_tokenizer/BUILD
new file mode 100644
index 0000000..59883fc
--- /dev/null
+++ b/pw_tokenizer/BUILD
@@ -0,0 +1,162 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+load(
+ "//pw_build:pigweed.bzl",
+ "pw_cc_binary",
+ "pw_cc_library",
+ "pw_cc_test",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"]) # Apache License 2.0
+
+pw_cc_library(
+ name = "pw_tokenizer",
+ srcs = [
+ "public/pw_tokenizer/config.h",
+ "public/pw_tokenizer/internal/argument_types.h",
+ "public/pw_tokenizer/internal/argument_types_macro_4_byte.h",
+ "public/pw_tokenizer/internal/argument_types_macro_8_byte.h",
+ "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h",
+ "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h",
+ "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h",
+ "public/pw_tokenizer/internal/tokenize_string.h",
+ "tokenize.cc",
+ ],
+ hdrs = [
+ "public/pw_tokenizer/pw_tokenizer_65599_fixed_length_hash.h",
+ "public/pw_tokenizer/tokenize.h",
+ ],
+ includes = ["public"],
+ deps = [
+ "//pw_preprocessor",
+ "//pw_span",
+ "//pw_varint",
+ ],
+)
+
+pw_cc_library(
+ name = "decoder",
+ srcs = [
+ "decode.cc",
+ "detokenize.cc",
+ "token_database.cc",
+ ],
+ hdrs = [
+ "public/pw_tokenizer/detokenize.h",
+ "public/pw_tokenizer/internal/decode.h",
+ "public/pw_tokenizer/token_database.h",
+ ],
+ includes = ["public"],
+ deps = [
+ "//pw_span",
+ "//pw_varint",
+ ],
+)
+
+# Executable for generating test data for the C++ and Python detokenizers. This
+# target should only be built for the host.
+pw_cc_binary(
+ name = "generate_decoding_test_data",
+ srcs = [
+ "generate_decoding_test_data.cc",
+ ],
+ deps = [
+ ":decoder",
+ ":pw_tokenizer",
+ "//pw_preprocessor",
+ "//pw_varint",
+ ],
+)
+
+pw_cc_test(
+ name = "argument_types_test",
+ srcs = [
+ "argument_types_test.c",
+ "argument_types_test.cc",
+ "pw_tokenizer_private/argument_types_test.h",
+ ],
+ deps = [
+ ":pw_tokenizer",
+ ],
+)
+
+pw_cc_test(
+ name = "decode_test",
+ srcs = [
+ "decode_test.cc",
+ "pw_tokenizer_private/tokenized_string_decoding_test_data.h",
+ "pw_tokenizer_private/varint_decoding_test_data.h",
+ ],
+ deps = [
+ ":decoder",
+ "//pw_varint",
+ ],
+)
+
+pw_cc_test(
+ name = "detokenize_test",
+ srcs = [
+ "detokenize_test.cc",
+ ],
+ deps = [
+ ":decoder",
+ ],
+)
+
+pw_cc_test(
+ name = "hash_test",
+ srcs = [
+ "hash_test.cc",
+ "pw_tokenizer_private/generated_hash_test_cases.h",
+ ],
+ deps = [
+ ":pw_tokenizer",
+ ],
+)
+
+pw_cc_test(
+ name = "token_database_test",
+ srcs = [
+ "token_database_test.cc",
+ ],
+ deps = [
+ ":decoder",
+ ],
+)
+
+pw_cc_test(
+ name = "tokenize_test",
+ srcs = [
+ "pw_tokenizer_private/tokenize_test.h",
+ "tokenize_test.c",
+ "tokenize_test.cc",
+ ],
+ deps = [
+ ":pw_tokenizer",
+ "//pw_varint",
+ ],
+)
+
+# Create a shared library for the tokenizer JNI wrapper. The include paths for
+# the JNI headers must be available in the system or provided with the
+# pw_java_native_interface_include_dirs variable.
+filegroup(
+ name = "detokenizer_jni",
+ srcs = [
+ "java/dev/pigweed/tokenizer/detokenizer.cc",
+ ],
+)
diff --git a/pw_tokenizer/BUILD.gn b/pw_tokenizer/BUILD.gn
new file mode 100644
index 0000000..5567a7a
--- /dev/null
+++ b/pw_tokenizer/BUILD.gn
@@ -0,0 +1,213 @@
+# Copyright 2020 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+# https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+import("$dir_pw_docgen/docs.gni")
+import("$dir_pw_unit_test/test.gni")
+
+config("default_config") {
+ include_dirs = [ "public" ]
+}
+
+source_set("pw_tokenizer") {
+ public_configs = [
+ "$dir_pw_build:pw_default_cpp",
+ ":default_config",
+ ]
+ public_deps = [
+ "$dir_pw_preprocessor",
+ "$dir_pw_span",
+ ]
+ deps = [
+ "$dir_pw_varint",
+ ]
+ public = [
+ "public/pw_tokenizer/pw_tokenizer_65599_fixed_length_hash.h",
+ "public/pw_tokenizer/tokenize.h",
+ ]
+ sources = [
+ "public/pw_tokenizer/config.h",
+ "public/pw_tokenizer/internal/argument_types.h",
+ "public/pw_tokenizer/internal/argument_types_macro_4_byte.h",
+ "public/pw_tokenizer/internal/argument_types_macro_8_byte.h",
+ "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h",
+ "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h",
+ "public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h",
+ "public/pw_tokenizer/internal/tokenize_string.h",
+ "tokenize.cc",
+ ]
+ sources += public
+ friend = [
+ ":argument_types_test",
+ ":hash_test",
+ ]
+}
+
+source_set("decoder") {
+ public_configs = [
+ "$dir_pw_build:pw_default_cpp",
+ ":default_config",
+ ]
+ public_deps = [
+ "$dir_pw_span",
+ ]
+ deps = [
+ "$dir_pw_varint",
+ ]
+ public = [
+ "public/pw_tokenizer/detokenize.h",
+ "public/pw_tokenizer/token_database.h",
+ ]
+ sources = [
+ "decode.cc",
+ "detokenize.cc",
+ "public/pw_tokenizer/internal/decode.h",
+ "token_database.cc",
+ ]
+ sources += public
+ friend = [
+ ":decode_test",
+ ":generate_decoding_test_data",
+ ]
+}
+
+# Executable for generating test data for the C++ and Python detokenizers. This
+# target should only be built for the host.
+executable("generate_decoding_test_data") {
+ deps = [
+ ":decoder",
+ ":pw_tokenizer",
+ "$dir_pw_varint",
+ ]
+ sources = [
+ "generate_decoding_test_data.cc",
+ ]
+}
+
+pw_test_group("tests") {
+ tests = [
+ ":argument_types_test",
+ ":decode_test",
+ ":detokenize_test",
+ ":hash_test",
+ ":token_database_test",
+ ":tokenize_test",
+ ]
+ group_deps = [
+ "$dir_pw_preprocessor:tests",
+ "$dir_pw_span:tests",
+ "$dir_pw_status:tests",
+ ]
+}
+
+pw_test("argument_types_test") {
+ sources = [
+ "argument_types_test.c",
+ "argument_types_test.cc",
+ "pw_tokenizer_private/argument_types_test.h",
+ ]
+ deps = [
+ ":pw_tokenizer",
+ ]
+}
+
+pw_test("decode_test") {
+ sources = [
+ "decode_test.cc",
+ "pw_tokenizer_private/tokenized_string_decoding_test_data.h",
+ "pw_tokenizer_private/varint_decoding_test_data.h",
+ ]
+ deps = [
+ ":decoder",
+ "$dir_pw_varint",
+ ]
+}
+
+pw_test("detokenize_test") {
+ sources = [
+ "detokenize_test.cc",
+ ]
+ deps = [
+ ":decoder",
+ ]
+}
+
+pw_test("hash_test") {
+ sources = [
+ "hash_test.cc",
+ "pw_tokenizer_private/generated_hash_test_cases.h",
+ ]
+ deps = [
+ ":pw_tokenizer",
+ ]
+}
+
+pw_test("token_database_test") {
+ sources = [
+ "token_database_test.cc",
+ ]
+ deps = [
+ ":decoder",
+ ]
+}
+
+pw_test("tokenize_test") {
+ sources = [
+ "pw_tokenizer_private/tokenize_test.h",
+ "tokenize_test.c",
+ "tokenize_test.cc",
+ ]
+ deps = [
+ ":pw_tokenizer",
+ "$dir_pw_varint",
+ ]
+}
+
+declare_args() {
+ # pw_java_native_interface_include_dirs specifies the paths to use for
+ # building Java Native Interface libraries. If no paths are provided, targets
+ # that require JNI may not build correctly.
+ #
+ # Example JNI include paths for a Linux system:
+ #
+ # pw_java_native_interface_include_dirs = [
+ # "/usr/local/buildtools/java/jdk/include/",
+ # "/usr/local/buildtools/java/jdk/include/linux",
+ # ]
+ #
+ pw_java_native_interface_include_dirs = []
+}
+
+# Create a shared library for the tokenizer JNI wrapper. The include paths for
+# the JNI headers must be available in the system or provided with the
+# pw_java_native_interface_include_dirs variable.
+shared_library("detokenizer_jni") {
+ public_configs = [
+ "$dir_pw_build:pw_default_cpp",
+ ":default_config",
+ ]
+ include_dirs = pw_java_native_interface_include_dirs
+ sources = [
+ "java/dev/pigweed/tokenizer/detokenizer.cc",
+ ]
+ public_deps = [
+ ":decoder",
+ "$dir_pw_preprocessor",
+ ]
+}
+
+pw_doc_group("docs") {
+ sources = [
+ "docs.rst",
+ ]
+}
diff --git a/pw_tokenizer/argument_types_test.c b/pw_tokenizer/argument_types_test.c
new file mode 100644
index 0000000..08ec027
--- /dev/null
+++ b/pw_tokenizer/argument_types_test.c
@@ -0,0 +1,102 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// This C source file tests that the tokenizer argument type encoding works
+// correctly in C. These functions are called from the main C++ test file
+// argument_types_test.cc.
+#include "pw_tokenizer_private/argument_types_test.h"
+
+#include <assert.h>
+#include <stddef.h>
+
+#ifdef __cplusplus
+#error "This is a test of C code and must be compiled as C, not C++."
+#endif // __cplusplus
+
+struct DummyType {}; // stand-in type for pointer argument type test
+
+// Check each relevant type mapping using static_asserts.
+#define CHECK_TYPE(c_type, enum_type) \
+ static_assert(_PW_VARARGS_TYPE((c_type)1) == enum_type, \
+ #c_type " should map to " #enum_type)
+
+// integral
+// clang-format off
+CHECK_TYPE(_Bool, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(char, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(signed char, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(unsigned char, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(short, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(unsigned short, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(int, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(unsigned int, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(long, _PW_TOKENIZER_SELECT_INT_TYPE(long));
+CHECK_TYPE(unsigned long, _PW_TOKENIZER_SELECT_INT_TYPE(unsigned long));
+CHECK_TYPE(long long, PW_TOKENIZER_ARG_TYPE_INT64);
+CHECK_TYPE(unsigned long long, PW_TOKENIZER_ARG_TYPE_INT64);
+
+// floating point
+CHECK_TYPE(float, PW_TOKENIZER_ARG_TYPE_DOUBLE);
+CHECK_TYPE(double, PW_TOKENIZER_ARG_TYPE_DOUBLE);
+CHECK_TYPE(long double, PW_TOKENIZER_ARG_TYPE_DOUBLE);
+
+// strings
+CHECK_TYPE(char*, PW_TOKENIZER_ARG_TYPE_STRING);
+CHECK_TYPE(const char*, PW_TOKENIZER_ARG_TYPE_STRING);
+
+// pointers (which should map to the appropriate sized integer)
+CHECK_TYPE(void*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(const void*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(signed char*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(unsigned char*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(int*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(long long*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(struct DummyType*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+// clang-format on
+
+// null
+static_assert(_PW_VARARGS_TYPE(NULL) == _PW_TOKENIZER_SELECT_INT_TYPE(void*),
+ "");
+
+static char char_array[16];
+
+// Define the test functions that are called by the C++ unit test.
+#define DEFINE_TEST_FUNCTION(name, ...) \
+ pw_TokenizerArgTypes pw_TestTokenizer##name(void) { \
+ (void)char_array; \
+ return PW_TOKENIZER_ARG_TYPES(__VA_ARGS__); \
+ }
+
+DEFINE_TEST_FUNCTION(NoArgs);
+
+DEFINE_TEST_FUNCTION(Char, 'a');
+DEFINE_TEST_FUNCTION(Uint8, ((uint8_t)23));
+DEFINE_TEST_FUNCTION(Uint16, ((int16_t)100));
+DEFINE_TEST_FUNCTION(Int32, ((int32_t)1));
+DEFINE_TEST_FUNCTION(Int64, ((int64_t)0));
+DEFINE_TEST_FUNCTION(Uint64, ((uint64_t)1));
+DEFINE_TEST_FUNCTION(Float, 1e10f)
+DEFINE_TEST_FUNCTION(Double, -2.5e-50);
+DEFINE_TEST_FUNCTION(String, "const char*");
+DEFINE_TEST_FUNCTION(MutableString, ((char*)NULL));
+
+DEFINE_TEST_FUNCTION(IntFloat, 54321, ((float)0));
+DEFINE_TEST_FUNCTION(Uint64Char, ((uint64_t)0ull), ((unsigned char)'x'));
+DEFINE_TEST_FUNCTION(StringString, char_array, ((const char*)NULL));
+DEFINE_TEST_FUNCTION(Uint16Int, ((uint16_t)100), ((int)0));
+DEFINE_TEST_FUNCTION(FloatString, 100.0f, "string");
+
+DEFINE_TEST_FUNCTION(Null, NULL);
+DEFINE_TEST_FUNCTION(Pointer, ((void*)NULL));
+DEFINE_TEST_FUNCTION(PointerPointer, (int*)char_array, (void*)0);
diff --git a/pw_tokenizer/argument_types_test.cc b/pw_tokenizer/argument_types_test.cc
new file mode 100644
index 0000000..a4d2676
--- /dev/null
+++ b/pw_tokenizer/argument_types_test.cc
@@ -0,0 +1,275 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_tokenizer/internal/argument_types.h"
+
+#include <cstddef>
+
+#include "gtest/gtest.h"
+#include "pw_preprocessor/concat.h"
+#include "pw_tokenizer_private/argument_types_test.h"
+
+namespace pw::tokenizer {
+namespace {
+
+struct DummyType {};
+
+// Check each relevant type mapping.
+#define CHECK_TYPE(c_type, enum_type) \
+ static_assert(_PW_VARARGS_TYPE((c_type)0) == enum_type, \
+ #c_type " should map to " #enum_type)
+
+// integral
+// clang-format off
+CHECK_TYPE(bool, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(char, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(signed char, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(unsigned char, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(short, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(unsigned short, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(int, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(unsigned int, PW_TOKENIZER_ARG_TYPE_INT);
+CHECK_TYPE(long, _PW_TOKENIZER_SELECT_INT_TYPE(long));
+CHECK_TYPE(unsigned long, _PW_TOKENIZER_SELECT_INT_TYPE(unsigned long));
+CHECK_TYPE(long long, PW_TOKENIZER_ARG_TYPE_INT64);
+CHECK_TYPE(unsigned long long, PW_TOKENIZER_ARG_TYPE_INT64);
+
+// floating point
+CHECK_TYPE(float, PW_TOKENIZER_ARG_TYPE_DOUBLE);
+CHECK_TYPE(double, PW_TOKENIZER_ARG_TYPE_DOUBLE);
+CHECK_TYPE(long double, PW_TOKENIZER_ARG_TYPE_DOUBLE);
+
+// strings
+CHECK_TYPE(char*, PW_TOKENIZER_ARG_TYPE_STRING);
+CHECK_TYPE(const char*, PW_TOKENIZER_ARG_TYPE_STRING);
+
+// pointers (which should map to the appropriate sized integer)
+CHECK_TYPE(void*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(const void*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(signed char*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(unsigned char*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(int*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(long long*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+CHECK_TYPE(DummyType*, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+
+// nullptr
+CHECK_TYPE(std::nullptr_t, _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+static_assert(_PW_VARARGS_TYPE(nullptr) ==
+ _PW_TOKENIZER_SELECT_INT_TYPE(void*));
+
+// clang-format on
+
+// Define a macro that generates expected values for tests. This works with
+// either 4-bit or 6-bit argument counts (for encoding types in a uint32_t or
+// uint64_t).
+#define PACKED_TYPES(...) \
+ ((PW_CONCAT(0b, __VA_ARGS__, u) << PW_TOKENIZER_TYPE_COUNT_SIZE_BITS) | \
+ PW_ARG_COUNT(__VA_ARGS__))
+
+// Test this test macro for both uint32_t and uint64_t.
+#if PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES == 4
+
+static_assert(PACKED_TYPES(00) == 0b00'0001u);
+static_assert(PACKED_TYPES(11) == 0b11'0001u);
+static_assert(PACKED_TYPES(01, 10) == 0b0110'0010u);
+static_assert(PACKED_TYPES(11, 01, 10) == 0b110110'0011u);
+static_assert(PACKED_TYPES(11, 10, 01, 00) == 0b11'10'01'00'0100u);
+
+#elif PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES == 8
+
+static_assert(PACKED_TYPES(00) == 0b00'000001u);
+static_assert(PACKED_TYPES(11) == 0b11'000001u);
+static_assert(PACKED_TYPES(01, 10) == 0b0110'000010u);
+static_assert(PACKED_TYPES(11, 01, 10) == 0b110110'000011u);
+static_assert(PACKED_TYPES(11, 10, 01, 00) == 0b11'10'01'00'000100u);
+
+#else
+
+#error "Unsupported value for PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES"
+
+#endif // PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES
+
+#define SOME_OTHER_MACRO(...) PW_TOKENIZER_ARG_TYPES(__VA_ARGS__)
+
+TEST(ArgumentTypes, Empty) {
+ static_assert(PW_TOKENIZER_ARG_TYPES() == 0u);
+ static_assert(PW_TOKENIZER_ARG_TYPES(/* nothing here */) == 0u);
+ static_assert(SOME_OTHER_MACRO() == 0u);
+ static_assert(SOME_OTHER_MACRO(/* nothing here */) == 0u);
+}
+
+TEST(ArgumentTypes, Int32) {
+ static_assert(PW_TOKENIZER_ARG_TYPES(0) == PACKED_TYPES(00));
+ static_assert(PW_TOKENIZER_ARG_TYPES('a') == PACKED_TYPES(00));
+ static_assert(PW_TOKENIZER_ARG_TYPES(u'X') == PACKED_TYPES(00));
+ static_assert(PW_TOKENIZER_ARG_TYPES(static_cast<uint16_t>(1)) ==
+ PACKED_TYPES(00));
+}
+
+TEST(ArgumentTypes, Int64) {
+ static_assert(PW_TOKENIZER_ARG_TYPES(-123ll) == PACKED_TYPES(01));
+ static_assert(PW_TOKENIZER_ARG_TYPES(123ull) == PACKED_TYPES(01));
+ static_assert(PW_TOKENIZER_ARG_TYPES(static_cast<uint64_t>(1)) ==
+ PACKED_TYPES(01));
+}
+
+TEST(ArgumentTypes, Double) {
+ static_assert(PW_TOKENIZER_ARG_TYPES(1.0) == PACKED_TYPES(10));
+ static_assert(PW_TOKENIZER_ARG_TYPES(5.0f) == PACKED_TYPES(10));
+ float number = 9;
+ static_assert(PW_TOKENIZER_ARG_TYPES(number) == PACKED_TYPES(10));
+}
+
+TEST(ArgumentTypes, String) {
+ static_assert(PW_TOKENIZER_ARG_TYPES("string") == PACKED_TYPES(11));
+ const char buffer[2] = {'a', 'b'};
+ static_assert(PW_TOKENIZER_ARG_TYPES(buffer) == PACKED_TYPES(11));
+ char mutable_buffer[8] = {};
+ static_assert(PW_TOKENIZER_ARG_TYPES(mutable_buffer) == PACKED_TYPES(11));
+ char character = 'a';
+ static_assert(PW_TOKENIZER_ARG_TYPES(&character) == PACKED_TYPES(11));
+}
+
+TEST(ArgumentTypes, Pointer) {
+ static const bool some_data[8] = {};
+
+ // For 32-bit systems, non-string pointers show up as int32.
+ static_assert(sizeof(void*) != sizeof(int32_t) ||
+ PACKED_TYPES(00) == PW_TOKENIZER_ARG_TYPES(nullptr));
+ static_assert(sizeof(void*) != sizeof(int32_t) ||
+ PACKED_TYPES(00) ==
+ PW_TOKENIZER_ARG_TYPES(static_cast<void*>(nullptr)));
+ static_assert(sizeof(void*) != sizeof(int32_t) ||
+ PACKED_TYPES(00) == PW_TOKENIZER_ARG_TYPES(some_data));
+
+ // For 64-bit systems, non-string pointers show up as int64.
+ static_assert(sizeof(void*) != sizeof(int64_t) ||
+ PACKED_TYPES(01) == PW_TOKENIZER_ARG_TYPES(nullptr));
+ static_assert(sizeof(void*) != sizeof(int64_t) ||
+ PACKED_TYPES(01) ==
+ PW_TOKENIZER_ARG_TYPES(static_cast<void*>(nullptr)));
+ static_assert(sizeof(void*) != sizeof(int64_t) ||
+ PACKED_TYPES(01) == PW_TOKENIZER_ARG_TYPES(some_data));
+}
+
+TEST(ArgumentTypes, TwoArgs) {
+ static_assert(PW_TOKENIZER_ARG_TYPES(-100, 1000) == PACKED_TYPES(00, 00));
+ static_assert(PW_TOKENIZER_ARG_TYPES(-100, 10ll) == PACKED_TYPES(01, 00));
+ static_assert(PW_TOKENIZER_ARG_TYPES(-100, 1.0f) == PACKED_TYPES(10, 00));
+ static_assert(PW_TOKENIZER_ARG_TYPES(-100, "hi") == PACKED_TYPES(11, 00));
+
+ static_assert(PW_TOKENIZER_ARG_TYPES(1ull, 1000) == PACKED_TYPES(00, 01));
+ static_assert(PW_TOKENIZER_ARG_TYPES(1ull, 10ll) == PACKED_TYPES(01, 01));
+ static_assert(PW_TOKENIZER_ARG_TYPES(1ull, 1.0f) == PACKED_TYPES(10, 01));
+ static_assert(PW_TOKENIZER_ARG_TYPES(1ull, "hi") == PACKED_TYPES(11, 01));
+
+ static_assert(PW_TOKENIZER_ARG_TYPES(9.0f, 1000) == PACKED_TYPES(00, 10));
+ static_assert(PW_TOKENIZER_ARG_TYPES(9.0f, 10ll) == PACKED_TYPES(01, 10));
+ static_assert(PW_TOKENIZER_ARG_TYPES(9.0f, 1.0f) == PACKED_TYPES(10, 10));
+ static_assert(PW_TOKENIZER_ARG_TYPES(9.0f, "hi") == PACKED_TYPES(11, 10));
+
+ static_assert(PW_TOKENIZER_ARG_TYPES("!!", 1000) == PACKED_TYPES(00, 11));
+ static_assert(PW_TOKENIZER_ARG_TYPES("!!", 10ll) == PACKED_TYPES(01, 11));
+ static_assert(PW_TOKENIZER_ARG_TYPES("!!", 1.0f) == PACKED_TYPES(10, 11));
+ static_assert(PW_TOKENIZER_ARG_TYPES("!!", "hi") == PACKED_TYPES(11, 11));
+}
+
+TEST(ArgumentTypes, MultipleArgs) {
+ // clang-format off
+ static_assert(PW_TOKENIZER_ARG_TYPES(1) == 1); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2) == 2); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3) == 3); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4) == 4); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5) == 5); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6) == 6); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7) == 7); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8) == 8); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9) == 9); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10) == 10); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11) == 11); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12) == 12); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13) == 13); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14) == 14); // NOLINT
+
+#if PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES >= 8
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14) == 14); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15) == 15); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16) == 16); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17) == 17); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18) == 18); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19) == 19); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20) == 20); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21) == 21); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22) == 22); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23) == 23); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24) == 24); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25) == 25); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26) == 26); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27) == 27); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28) == 28); // NOLINT
+ static_assert(PW_TOKENIZER_ARG_TYPES(1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29) == 29); // NOLINT
+#endif // PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES
+ // clang-format on
+}
+
+// The ArgumentTypesFromC test suite tests the arguments types macro in C. The
+// pw_Test* functions are defined in argument_types_c_test.c
+TEST(ArgumentTypesFromC, NoArgs) {
+ EXPECT_EQ(0b0000u, pw_TestTokenizerNoArgs());
+}
+
+TEST(ArgumentTypesFromC, OneArg_Int32) {
+ EXPECT_EQ(PACKED_TYPES(00), pw_TestTokenizerChar());
+ EXPECT_EQ(PACKED_TYPES(00), pw_TestTokenizerUint8());
+ EXPECT_EQ(PACKED_TYPES(00), pw_TestTokenizerUint16());
+ EXPECT_EQ(PACKED_TYPES(00), pw_TestTokenizerInt32());
+}
+
+TEST(ArgumentTypesFromC, OneArg_Int64) {
+ EXPECT_EQ(PACKED_TYPES(01), pw_TestTokenizerInt64());
+ EXPECT_EQ(PACKED_TYPES(01), pw_TestTokenizerUint64());
+}
+
+TEST(ArgumentTypesFromC, OneArg_Float) {
+ EXPECT_EQ(PACKED_TYPES(10), pw_TestTokenizerFloat());
+ EXPECT_EQ(PACKED_TYPES(10), pw_TestTokenizerDouble());
+}
+
+TEST(ArgumentTypesFromC, OneArg_String) {
+ EXPECT_EQ(PACKED_TYPES(11), pw_TestTokenizerString());
+ EXPECT_EQ(PACKED_TYPES(11), pw_TestTokenizerMutableString());
+}
+
+TEST(ArgumentTypesFromC, MultipleArgs) {
+ EXPECT_EQ(PACKED_TYPES(10, 00), pw_TestTokenizerIntFloat());
+ EXPECT_EQ(PACKED_TYPES(00, 01), pw_TestTokenizerUint64Char());
+ EXPECT_EQ(PACKED_TYPES(11, 11), pw_TestTokenizerStringString());
+ EXPECT_EQ(PACKED_TYPES(00, 00), pw_TestTokenizerUint16Int());
+ EXPECT_EQ(PACKED_TYPES(11, 10), pw_TestTokenizerFloatString());
+}
+
+TEST(ArgumentTypesFromC, Pointers) {
+ if constexpr (sizeof(void*) == sizeof(int32_t)) {
+ EXPECT_EQ(PACKED_TYPES(00), pw_TestTokenizerNull());
+ EXPECT_EQ(PACKED_TYPES(00), pw_TestTokenizerPointer());
+ EXPECT_EQ(PACKED_TYPES(00, 00), pw_TestTokenizerPointerPointer());
+ } else { // 64-bit system
+ EXPECT_EQ(PACKED_TYPES(01), pw_TestTokenizerNull());
+ EXPECT_EQ(PACKED_TYPES(01), pw_TestTokenizerPointer());
+ EXPECT_EQ(PACKED_TYPES(01, 01), pw_TestTokenizerPointerPointer());
+ }
+}
+
+} // namespace
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/decode.cc b/pw_tokenizer/decode.cc
new file mode 100644
index 0000000..22ce61a
--- /dev/null
+++ b/pw_tokenizer/decode.cc
@@ -0,0 +1,353 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_tokenizer/internal/decode.h"
+
+#include <algorithm>
+#include <array>
+#include <cctype>
+#include <cstring>
+
+#include "pw_varint/varint.h"
+
+namespace pw::tokenizer {
+namespace {
+
+// Functions for parsing a printf format specifier.
+size_t SkipFlags(const char* str) {
+ size_t i = 0;
+ while (str[i] == '-' || str[i] == '+' || str[i] == '#' || str[i] == ' ' ||
+ str[i] == '0') {
+ i += 1;
+ }
+ return i;
+}
+
+size_t SkipAsteriskOrInteger(const char* str) {
+ if (str[0] == '*') {
+ return 1;
+ }
+
+ size_t i = (str[0] == '-' || str[0] == '+') ? 1 : 0;
+
+ while (std::isdigit(str[i])) {
+ i += 1;
+ }
+ return i;
+}
+
+std::array<char, 2> ReadLengthModifier(const char* str) {
+ // Check for ll or hh.
+ if (str[0] == str[1] && (str[0] == 'l' || str[0] == 'h')) {
+ return {str[0], str[1]};
+ }
+ if (std::strchr("hljztL", str[0]) != nullptr) {
+ return {str[0]};
+ }
+ return {};
+}
+
+// Returns the error message that is used in place of a decoded arg when an
+// error occurs.
+std::string ErrorMessage(ArgStatus status,
+ const std::string_view& spec,
+ const std::string_view& value) {
+ const char* message;
+ if (status.HasError(ArgStatus::kSkipped)) {
+ message = "SKIPPED";
+ } else if (status.HasError(ArgStatus::kMissing)) {
+ message = "MISSING";
+ } else if (status.HasError(ArgStatus::kDecodeError)) {
+ message = "ERROR";
+ } else {
+ message = "INTERNAL ERROR";
+ }
+
+ std::string result(PW_TOKENIZER_ARG_DECODING_ERROR_PREFIX);
+ result.append(spec);
+ result.push_back(' ');
+ result.append(message);
+
+ if (!value.empty()) {
+ result.push_back(' ');
+ result.push_back('(');
+ result.append(value);
+ result.push_back(')');
+ }
+
+ result.append(PW_TOKENIZER_ARG_DECODING_ERROR_SUFFIX);
+ return result;
+}
+
+} // namespace
+
+DecodedArg::DecodedArg(ArgStatus error,
+ const std::string_view& spec,
+ size_t raw_size_bytes,
+ const std::string_view& value)
+ : value_(ErrorMessage(error, spec, value)),
+ spec_(spec),
+ raw_data_size_bytes_(raw_size_bytes),
+ status_(error) {}
+
+StringSegment StringSegment::ParseFormatSpec(const char* format) {
+ if (format[0] != '%' || format[1] == '\0') {
+ return StringSegment();
+ }
+
+ // Parse the format specifier.
+ size_t i = 1;
+
+ // Skip the flags.
+ i += SkipFlags(&format[i]);
+
+ // Skip the field width.
+ i += SkipAsteriskOrInteger(&format[i]);
+
+ // Skip the precision.
+ if (format[i] == '.') {
+ i += 1;
+ i += SkipAsteriskOrInteger(&format[i]);
+ }
+
+ // Read the length modifier.
+ const std::array<char, 2> length = ReadLengthModifier(&format[i]);
+ i += (length[0] == '\0' ? 0 : 1) + (length[1] == '\0' ? 0 : 1);
+
+ // Read the conversion specifier.
+ const char spec = format[i];
+
+ Type type;
+ if (spec == 's') {
+ type = kString;
+ } else if (spec == 'c' || spec == 'd' || spec == 'i') {
+ type = kSignedInt;
+ } else if (std::strchr("oxXup", spec) != nullptr) {
+ // The source size matters for unsigned integers because they need to be
+ // masked off to their correct length, since zig-zag decode sign extends.
+ // TODO(hepler): 64-bit targets likely have 64-bit l, j, z, and t. Also, p
+ // needs to be 64-bit on these targets.
+ type = length[0] == 'j' || length[1] == 'l' ? kUnsigned64 : kUnsigned32;
+ } else if (std::strchr("fFeEaAgG", spec) != nullptr) {
+ type = kFloatingPoint;
+ } else if (spec == '%' && i == 1) {
+ type = kPercent;
+ } else {
+ return StringSegment();
+ }
+
+ return {std::string_view(format, i + 1), type, VarargSize(length, spec)};
+}
+
+StringSegment::ArgSize StringSegment::VarargSize(std::array<char, 2> length,
+ char spec) {
+ // Use pointer size for %p or any other type (for which this doesn't matter).
+ if (std::strchr("cdioxXu", spec) == nullptr) {
+ return VarargSize<void*>();
+ }
+ if (length[0] == 'l') {
+ return length[1] == 'l' ? VarargSize<long long>() : VarargSize<long>();
+ }
+ if (length[0] == 'j') {
+ return VarargSize<intmax_t>();
+ }
+ if (length[0] == 'z') {
+ return VarargSize<size_t>();
+ }
+ if (length[0] == 't') {
+ return VarargSize<ptrdiff_t>();
+ }
+ return VarargSize<int>();
+}
+
+DecodedArg StringSegment::DecodeString(
+ const span<const uint8_t>& arguments) const {
+ if (arguments.empty()) {
+ return DecodedArg(ArgStatus::kMissing, text_);
+ }
+
+ ArgStatus status =
+ (arguments[0] & 0x80u) == 0u ? ArgStatus::kOk : ArgStatus::kTruncated;
+
+ const uint_fast8_t size = arguments[0] & 0x7Fu;
+
+ if (arguments.size() - 1 < size) {
+ status.Update(ArgStatus::kDecodeError);
+ return DecodedArg(
+ status,
+ text_,
+ arguments.size(),
+ {reinterpret_cast<const char*>(&arguments[1]), arguments.size() - 1});
+ }
+
+ std::string value(reinterpret_cast<const char*>(&arguments[1]), size);
+
+ if (status.HasError(ArgStatus::kTruncated)) {
+ value.append("[...]");
+ }
+
+ return DecodedArg::FromValue(text_.c_str(), value.c_str(), 1 + size, status);
+}
+
+DecodedArg StringSegment::DecodeInteger(
+ const span<const uint8_t>& arguments) const {
+ if (arguments.empty()) {
+ return DecodedArg(ArgStatus::kMissing, text_);
+ }
+
+ int64_t value;
+ const size_t bytes = varint::Decode(pw::as_bytes(arguments), &value);
+
+ if (bytes == 0u) {
+ return DecodedArg(ArgStatus::kDecodeError,
+ text_,
+ std::min(varint::kMaxVarintSizeBytes, arguments.size()));
+ }
+
+ // Unsigned ints need to be masked to their bit width due to sign extension.
+ if (type_ == kUnsigned32) {
+ value &= 0xFFFFFFFFu;
+ }
+
+ if (local_size_ == k32Bit) {
+ return DecodedArg::FromValue(
+ text_.c_str(), static_cast<uint32_t>(value), bytes);
+ }
+ return DecodedArg::FromValue(text_.c_str(), value, bytes);
+}
+
+DecodedArg StringSegment::DecodeFloatingPoint(
+ const span<const uint8_t>& arguments) const {
+ static_assert(sizeof(float) == 4u);
+ if (arguments.size() < sizeof(float)) {
+ return DecodedArg(ArgStatus::kMissing, text_);
+ }
+
+ float value;
+ std::memcpy(&value, arguments.data(), sizeof(value));
+ return DecodedArg::FromValue(text_.c_str(), value, sizeof(value));
+}
+
+DecodedArg StringSegment::Decode(const span<const uint8_t>& arguments) const {
+ switch (type_) {
+ case kLiteral:
+ return DecodedArg(text_);
+ case kPercent:
+ return DecodedArg("%");
+ case kString:
+ return DecodeString(arguments);
+ case kSignedInt:
+ case kUnsigned32:
+ case kUnsigned64:
+ return DecodeInteger(arguments);
+ case kFloatingPoint:
+ return DecodeFloatingPoint(arguments);
+ }
+
+ return DecodedArg(ArgStatus::kDecodeError, text_);
+}
+
+DecodedArg StringSegment::Skip() const {
+ switch (type_) {
+ case kLiteral:
+ return DecodedArg(text_);
+ case kPercent:
+ return DecodedArg("%");
+ default:
+ return DecodedArg(ArgStatus::kSkipped, text_);
+ }
+}
+
+std::string DecodedFormatString::value() const {
+ std::string output;
+
+ for (const DecodedArg& arg : segments_) {
+ output.append(arg.ok() ? arg.value() : arg.spec());
+ }
+
+ return output;
+}
+
+std::string DecodedFormatString::value_with_errors() const {
+ std::string output;
+
+ for (const DecodedArg& arg : segments_) {
+ output.append(arg.value());
+ }
+
+ return output;
+}
+
+size_t DecodedFormatString::argument_count() const {
+ return std::count_if(segments_.begin(), segments_.end(), [](const auto& arg) {
+ return !arg.spec().empty();
+ });
+}
+
+size_t DecodedFormatString::decoding_errors() const {
+ return std::count_if(segments_.begin(), segments_.end(), [](const auto& arg) {
+ return !arg.ok();
+ });
+}
+
+FormatString::FormatString(const char* format) {
+ const char* text_start = format;
+
+ while (format[0] != '\0') {
+ if (StringSegment spec = StringSegment::ParseFormatSpec(format);
+ !spec.empty()) {
+ // Add the text segment seen so far (if any).
+ if (text_start < format) {
+ segments_.emplace_back(
+ std::string_view(text_start, format - text_start));
+ }
+
+ // Move along the index and text segment start.
+ format += spec.text().size();
+ text_start = format;
+
+ // Add the format specifier that was just found.
+ segments_.push_back(std::move(spec));
+ } else {
+ format += 1;
+ }
+ }
+
+ if (text_start < format) {
+ segments_.emplace_back(std::string_view(text_start, format - text_start));
+ }
+}
+
+DecodedFormatString FormatString::Format(span<const uint8_t> arguments) const {
+ std::vector<DecodedArg> results;
+ bool skip = false;
+
+ for (const auto& segment : segments_) {
+ if (skip) {
+ results.push_back(segment.Skip());
+ } else {
+ results.push_back(segment.Decode(arguments));
+ arguments = arguments.subspan(results.back().raw_size_bytes());
+
+ // If an error occurred, skip decoding the remaining arguments.
+ if (!results.back().ok()) {
+ skip = true;
+ }
+ }
+ }
+
+ return DecodedFormatString(std::move(results), arguments.size());
+}
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/decode_test.cc b/pw_tokenizer/decode_test.cc
new file mode 100644
index 0000000..d379060
--- /dev/null
+++ b/pw_tokenizer/decode_test.cc
@@ -0,0 +1,242 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_tokenizer/internal/decode.h"
+
+#include <cstdio>
+#include <string>
+#include <string_view>
+
+#include "gtest/gtest.h"
+#include "pw_tokenizer_private/tokenized_string_decoding_test_data.h"
+#include "pw_tokenizer_private/varint_decoding_test_data.h"
+#include "pw_varint/varint.h"
+
+#define ERR PW_TOKENIZER_ARG_DECODING_ERROR
+
+namespace pw::tokenizer {
+namespace {
+
+using namespace std::literals::string_view_literals;
+
+const FormatString kOneArg("Hello %s");
+const FormatString kTwoArgs("The %d %s");
+
+const bool kSupportsC99Printf = []() {
+ char buf[16] = {};
+ std::snprintf(buf, sizeof(buf), "%zu", sizeof(char));
+ return buf[0] == '1' && buf[1] == '\0';
+}();
+
+const bool kSupportsFloatPrintf = []() {
+ char buf[16] = {};
+ std::snprintf(buf, sizeof(buf), "%.1f", 3.5f);
+ return buf[0] == '3' && buf[1] == '.' && buf[2] == '5' && buf[3] == '\0';
+}();
+
+bool FormatIsSupported(std::string_view value) {
+ return (kSupportsC99Printf || (value.find("%hh") == std::string_view::npos &&
+ value.find("%ll") == std::string_view::npos &&
+ value.find("%j") == std::string_view::npos &&
+ value.find("%z") == std::string_view::npos &&
+ value.find("%t") == std::string_view::npos)) &&
+ // To make absolutely certain there are no floating point numbers, skip
+ // anything with an f or e in it.
+ (kSupportsFloatPrintf || (value.find('f') == std::string_view::npos &&
+ value.find('e') == std::string_view::npos));
+}
+
+TEST(TokenizedStringDecode, TokenizedStringDecodingTestCases) {
+ const auto& test_data = test::tokenized_string_decoding::kTestData;
+ static_assert(sizeof(test_data) / sizeof(*test_data) > 100u);
+
+ size_t skipped = 0;
+
+ for (const auto& [format, expected, args] : test_data) {
+ if (FormatIsSupported(format)) {
+ ASSERT_EQ(FormatString(format).Format(args).value_with_errors(),
+ expected);
+ } else {
+ skipped += 1;
+ }
+ }
+
+ if (sizeof(void*) == 8) { // 64-bit systems should have full snprintf.
+ ASSERT_EQ(skipped, 0u);
+ }
+}
+
+TEST(TokenizedStringDecode, FullyDecodeInput_ZeroRemainingBytes) {
+ auto result = kOneArg.Format("\5hello");
+ EXPECT_EQ(result.value(), "Hello hello");
+ EXPECT_EQ(result.remaining_bytes(), 0u);
+ EXPECT_EQ(result.decoding_errors(), 0u);
+}
+
+TEST(TokenizedStringDecode, PartiallyDecodeInput_HasRemainingBytes) {
+ auto result = kOneArg.Format("\5helloworld");
+ EXPECT_EQ(result.value(), "Hello hello");
+ EXPECT_EQ(result.remaining_bytes(), 5u);
+ EXPECT_EQ(result.decoding_errors(), 0u);
+}
+
+TEST(TokenizedStringDecode, Truncation_NotAnError) {
+ auto result = kTwoArgs.Format("\6\x89musketeer");
+ EXPECT_EQ(result.value(), "The 3 musketeer[...]");
+ EXPECT_EQ(result.value_with_errors(), "The 3 musketeer[...]");
+ EXPECT_EQ(result.remaining_bytes(), 0u);
+ EXPECT_EQ(result.decoding_errors(), 0u);
+}
+
+TEST(TokenizedStringDecode, WrongStringLenth_IsErrorAndConsumesRestOfString) {
+ auto result = kTwoArgs.Format("\6\x0amusketeer");
+ EXPECT_EQ(result.value(), "The 3 %s");
+ EXPECT_EQ(result.value_with_errors(), "The 3 " ERR("%s ERROR (musketeer)"));
+ EXPECT_EQ(result.remaining_bytes(), 0u);
+ EXPECT_EQ(result.decoding_errors(), 1u);
+}
+
+TEST(TokenizedStringDecode, UnterminatedVarint_IsError) {
+ auto result = kTwoArgs.Format("\x80");
+ EXPECT_EQ(result.value(), "The %d %s");
+ EXPECT_EQ(result.value_with_errors(),
+ "The " ERR("%d ERROR") " " ERR("%s SKIPPED"));
+ EXPECT_EQ(result.remaining_bytes(), 0u);
+ EXPECT_EQ(result.decoding_errors(), 2u);
+}
+
+TEST(TokenizedStringDecode, UnterminatedVarint_ConsumesUpToMaxVarintSize) {
+ std::string_view data = "\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80";
+ auto result = kTwoArgs.Format(data);
+ EXPECT_EQ(result.value(), "The %d %s");
+ EXPECT_EQ(result.value_with_errors(),
+ "The " ERR("%d ERROR") " " ERR("%s SKIPPED"));
+ EXPECT_EQ(result.remaining_bytes(),
+ data.size() - varint::kMaxVarintSizeBytes);
+ EXPECT_EQ(result.decoding_errors(), 2u);
+}
+
+TEST(TokenizedStringDecode, MissingArguments_IsDecodeError) {
+ auto result = kTwoArgs.Format("");
+ EXPECT_EQ(result.value(), "The %d %s");
+ EXPECT_EQ(result.value_with_errors(),
+ "The " ERR("%d MISSING") " " ERR("%s SKIPPED"));
+ EXPECT_EQ(result.remaining_bytes(), 0u);
+ EXPECT_EQ(result.decoding_errors(), 2u);
+}
+
+TEST(VarintDecode, VarintDecodeTestCases) {
+ const auto& test_data = test::varint_decoding::kTestData;
+ static_assert(sizeof(test_data) / sizeof(*test_data) > 100u);
+
+ size_t skipped = 0;
+
+ for (const auto& [d_fmt, d_expected, u_fmt, u_expected, data] : test_data) {
+ if (FormatIsSupported(d_fmt)) {
+ ASSERT_EQ(FormatString(d_fmt).Format(data).value(), d_expected);
+ ASSERT_EQ(FormatString(u_fmt).Format(data).value(), u_expected);
+ } else {
+ skipped += 1;
+ }
+
+ if (sizeof(void*) == 8) { // 64-bit systems should have full snprintf.
+ ASSERT_EQ(skipped, 0u);
+ }
+ }
+}
+
+class DecodedFormatStringTest : public ::testing::Test {
+ protected:
+ DecodedFormatStringTest()
+ : no_args_("Give 110%% sometimes"),
+ one_arg_("so you can give %d%%"),
+ two_args_("%d.%d%% of the time") {}
+
+ const FormatString no_args_;
+ const FormatString one_arg_;
+ const FormatString two_args_;
+};
+
+TEST_F(DecodedFormatStringTest, Value) {
+ EXPECT_EQ("Give 110% sometimes", no_args_.Format("").value());
+ EXPECT_EQ("so you can give 0%", one_arg_.Format("\0"sv).value());
+ EXPECT_EQ("90.9% of the time", two_args_.Format("\xB4\x01\x12").value());
+}
+
+TEST_F(DecodedFormatStringTest, FormatSuccessfully_IsOk) {
+ EXPECT_TRUE(no_args_.Format("").ok());
+ EXPECT_TRUE(one_arg_.Format("\0"sv).ok());
+ EXPECT_TRUE(two_args_.Format("\xB4\x01\x12").ok());
+}
+
+TEST_F(DecodedFormatStringTest, FormatWithDecodingErrors_IsNotOkay) {
+ EXPECT_FALSE(one_arg_.Format("").ok());
+ EXPECT_FALSE(two_args_.Format("\x80").ok());
+}
+
+TEST_F(DecodedFormatStringTest, FormatWithRemainingBytes_IsNotOkay) {
+ EXPECT_FALSE(no_args_.Format("Hello").ok());
+ EXPECT_FALSE(one_arg_.Format("\0\0"sv).ok());
+ EXPECT_FALSE(two_args_.Format("\xB4\x01\x12?").ok());
+}
+
+TEST_F(DecodedFormatStringTest, FormatWithRemainingBytesAndError_IsNotOkay) {
+ EXPECT_FALSE(
+ one_arg_.Format("\x80\x80\x80\x80\x80\x80\x80\x80\x80\x80\x00"sv).ok());
+}
+
+TEST_F(DecodedFormatStringTest, RemainingBytes_NoData_IsZero) {
+ EXPECT_EQ(0u, no_args_.Format("").remaining_bytes());
+ EXPECT_EQ(0u, one_arg_.Format("").remaining_bytes());
+ EXPECT_EQ(0u, two_args_.Format("").remaining_bytes());
+}
+
+TEST_F(DecodedFormatStringTest, RemainingBytes_WithData_MatchesNumberOfArgs) {
+ EXPECT_EQ(2u, no_args_.Format("\x02\x02").remaining_bytes());
+ EXPECT_EQ(1u, one_arg_.Format("\x02\x02").remaining_bytes());
+ EXPECT_EQ(0u, two_args_.Format("\x02\x02").remaining_bytes());
+}
+
+TEST_F(DecodedFormatStringTest, ArgumentCount_NoData_MatchesNumberOfArgs) {
+ EXPECT_EQ(0u, no_args_.Format("").argument_count());
+ EXPECT_EQ(1u, one_arg_.Format("").argument_count());
+ EXPECT_EQ(2u, two_args_.Format("").argument_count());
+}
+
+TEST_F(DecodedFormatStringTest, ArgumentCount_WithData_MatchesNumberOfArgs) {
+ EXPECT_EQ(0u, no_args_.Format("\x02\x02").argument_count());
+ EXPECT_EQ(1u, one_arg_.Format("\x02\x02").argument_count());
+ EXPECT_EQ(2u, two_args_.Format("\x02\x02").argument_count());
+}
+
+TEST_F(DecodedFormatStringTest, DecodingErrors_NoData_MatchesArgumentCount) {
+ EXPECT_EQ(0u, no_args_.Format("").decoding_errors());
+ EXPECT_EQ(1u, one_arg_.Format("").decoding_errors());
+ EXPECT_EQ(2u, two_args_.Format("").decoding_errors());
+}
+
+TEST_F(DecodedFormatStringTest, DecodingErrors_OneArg_AllRemainingAreErrors) {
+ EXPECT_EQ(0u, no_args_.Format("\x02").decoding_errors());
+ EXPECT_EQ(0u, one_arg_.Format("\x02").decoding_errors());
+ EXPECT_EQ(1u, two_args_.Format("\x02").decoding_errors());
+}
+
+TEST_F(DecodedFormatStringTest, DecodingErrors_TwoArgs_IsZero) {
+ EXPECT_EQ(0u, no_args_.Format("\x02\x02").decoding_errors());
+ EXPECT_EQ(0u, one_arg_.Format("\x02\x02").decoding_errors());
+ EXPECT_EQ(0u, two_args_.Format("\x02\x02").decoding_errors());
+}
+
+} // namespace
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/detokenize.cc b/pw_tokenizer/detokenize.cc
new file mode 100644
index 0000000..ef46269
--- /dev/null
+++ b/pw_tokenizer/detokenize.cc
@@ -0,0 +1,125 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_tokenizer/detokenize.h"
+
+#include <algorithm>
+
+#include "pw_tokenizer/internal/decode.h"
+
+namespace pw::tokenizer {
+namespace {
+
+std::string UnknownTokenMessage(uint32_t value) {
+ std::string output(PW_TOKENIZER_ARG_DECODING_ERROR_PREFIX "unknown token ");
+
+ // Output a hexadecimal version of the token.
+ for (int shift = 28; shift >= 0; shift -= 4) {
+ output.push_back("0123456789abcdef"[(value >> shift) & 0xF]);
+ }
+
+ output.append(PW_TOKENIZER_ARG_DECODING_ERROR_SUFFIX);
+ return output;
+}
+
+// Decoding result with the date removed, for sorting.
+using DecodingResult = std::pair<DecodedFormatString, uint32_t>;
+
+// Determines if one result is better than the other if collisions occurred.
+// Returns true if lhs is preferred over rhs. This logic should match the
+// collision resolution logic in detokenize.py.
+bool IsBetterResult(const DecodingResult& lhs, const DecodingResult& rhs) {
+ // Favor the result for which decoding succeeded.
+ if (lhs.first.ok() != rhs.first.ok()) {
+ return lhs.first.ok();
+ }
+
+ // Favor the result for which all bytes were decoded.
+ if ((lhs.first.remaining_bytes() == 0u) !=
+ (rhs.first.remaining_bytes() == 0u)) {
+ return lhs.first.remaining_bytes() == 0u;
+ }
+
+ // Favor the result with fewer decoding errors.
+ if (lhs.first.decoding_errors() != rhs.first.decoding_errors()) {
+ return lhs.first.decoding_errors() < rhs.first.decoding_errors();
+ }
+
+ // Favor the result that successfully decoded the most arguments.
+ if (lhs.first.argument_count() != rhs.first.argument_count()) {
+ return lhs.first.argument_count() > rhs.first.argument_count();
+ }
+
+ // Favor the result that was removed from the database most recently.
+ return lhs.second > rhs.second;
+}
+
+} // namespace
+
+DetokenizedString::DetokenizedString(
+ uint32_t token,
+ const span<const TokenizedStringEntry>& entries,
+ const span<const uint8_t>& arguments)
+ : token_(token), has_token_(true) {
+ std::vector<DecodingResult> results;
+
+ for (const auto& [format, date_removed] : entries) {
+ results.push_back(DecodingResult{format.Format(arguments), date_removed});
+ }
+
+ std::sort(results.begin(), results.end(), IsBetterResult);
+
+ for (auto& result : results) {
+ matches_.push_back(std::move(result.first));
+ }
+}
+
+std::string DetokenizedString::BestString() const {
+ return matches_.empty() ? std::string() : matches_[0].value();
+}
+
+std::string DetokenizedString::BestStringWithErrors() const {
+ if (matches_.empty()) {
+ return has_token_ ? UnknownTokenMessage(token_)
+ : PW_TOKENIZER_ARG_DECODING_ERROR("missing token");
+ }
+ return matches_[0].value_with_errors();
+}
+
+Detokenizer::Detokenizer(const TokenDatabase& database) {
+ for (const auto& entry : database) {
+ database_[entry.token].emplace_back(entry.string, entry.date_removed);
+ }
+}
+
+DetokenizedString Detokenizer::Detokenize(
+ const span<const uint8_t>& encoded) const {
+ // The token is missing from the encoded data; there is nothing to do.
+ if (encoded.size() < sizeof(uint32_t)) {
+ return DetokenizedString();
+ }
+
+ const uint32_t token =
+ encoded[3] << 24 | encoded[2] << 16 | encoded[1] << 8 | encoded[0];
+
+ const auto result = database_.find(token);
+
+ return DetokenizedString(token,
+ result == database_.end()
+ ? span<TokenizedStringEntry>()
+ : span(result->second),
+ encoded.subspan(sizeof(token)));
+}
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/detokenize_test.cc b/pw_tokenizer/detokenize_test.cc
new file mode 100644
index 0000000..688cf7a
--- /dev/null
+++ b/pw_tokenizer/detokenize_test.cc
@@ -0,0 +1,289 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_tokenizer/detokenize.h"
+
+#include <string_view>
+
+#include "gtest/gtest.h"
+
+namespace pw::tokenizer {
+namespace {
+
+using namespace std::literals::string_view_literals;
+
+// Use a shorter name for the error string macro.
+#define ERR PW_TOKENIZER_ARG_DECODING_ERROR
+
+// Use alignas to ensure that the data is properly aligned to be read from a
+// token database entry struct. This avoids unaligned memory reads.
+alignas(TokenDatabase::RawEntry) constexpr char kBasicData[] =
+ "TOKENS\0\0"
+ "\x04\x00\x00\x00"
+ "\0\0\0\0"
+ "\x01\x00\x00\x00----"
+ "\x05\x00\x00\x00----"
+ "\xFF\x00\x00\x00----"
+ "\xFF\xEE\xEE\xDD----"
+ "One\0"
+ "TWO\0"
+ "333\0"
+ "FOUR";
+
+class Detokenize : public ::testing::Test {
+ protected:
+ Detokenize() : detok_(TokenDatabase::Create<kBasicData>()) {}
+ Detokenizer detok_;
+};
+
+TEST_F(Detokenize, NoFormatting) {
+ EXPECT_EQ(detok_.Detokenize("\1\0\0\0"sv).BestString(), "One");
+ EXPECT_EQ(detok_.Detokenize("\5\0\0\0"sv).BestString(), "TWO");
+ EXPECT_EQ(detok_.Detokenize("\xff\x00\x00\x00"sv).BestString(), "333");
+ EXPECT_EQ(detok_.Detokenize("\xff\xee\xee\xdd"sv).BestString(), "FOUR");
+}
+
+TEST_F(Detokenize, BestString_MissingToken_IsEmpty) {
+ EXPECT_FALSE(detok_.Detokenize("").ok());
+ EXPECT_TRUE(detok_.Detokenize("", 0u).BestString().empty());
+ EXPECT_TRUE(detok_.Detokenize("\1", 1u).BestString().empty());
+ EXPECT_TRUE(detok_.Detokenize("\1\0"sv).BestString().empty());
+ EXPECT_TRUE(detok_.Detokenize("\1\0\0"sv).BestString().empty());
+ EXPECT_TRUE(detok_.Detokenize("\0\0\0"sv).BestString().empty());
+}
+
+TEST_F(Detokenize, BestString_UnknownToken_IsEmpty) {
+ EXPECT_FALSE(detok_.Detokenize("\0\0\0\0"sv).ok());
+ EXPECT_TRUE(detok_.Detokenize("\0\0\0\0"sv).BestString().empty());
+ EXPECT_TRUE(detok_.Detokenize("\2\0\0\0"sv).BestString().empty());
+ EXPECT_TRUE(detok_.Detokenize("\x10\x32\x54\x76\x99"sv).BestString().empty());
+ EXPECT_TRUE(detok_.Detokenize("\x98\xba\xdc\xfe"sv).BestString().empty());
+}
+
+TEST_F(Detokenize, BestStringWithErrors_MissingToken_ErrorMessage) {
+ EXPECT_FALSE(detok_.Detokenize("").ok());
+ EXPECT_EQ(detok_.Detokenize("", 0u).BestStringWithErrors(),
+ ERR("missing token"));
+ EXPECT_EQ(detok_.Detokenize("\1", 1u).BestStringWithErrors(),
+ ERR("missing token"));
+ EXPECT_EQ(detok_.Detokenize("\1\0"sv).BestStringWithErrors(),
+ ERR("missing token"));
+ EXPECT_EQ(detok_.Detokenize("\1\0\0"sv).BestStringWithErrors(),
+ ERR("missing token"));
+ EXPECT_EQ(detok_.Detokenize("\0\0\0"sv).BestStringWithErrors(),
+ ERR("missing token"));
+}
+
+TEST_F(Detokenize, BestStringWithErrors_UnknownToken_ErrorMessage) {
+ EXPECT_FALSE(detok_.Detokenize("\0\0\0\0"sv).ok());
+ EXPECT_EQ(detok_.Detokenize("\0\0\0\0"sv).BestStringWithErrors(),
+ ERR("unknown token 00000000"));
+ EXPECT_EQ(detok_.Detokenize("\2\0\0\0"sv).BestStringWithErrors(),
+ ERR("unknown token 00000002"));
+ EXPECT_EQ(detok_.Detokenize("\x10\x32\x54\x76\x99"sv).BestStringWithErrors(),
+ ERR("unknown token 76543210"));
+ EXPECT_EQ(detok_.Detokenize("\x98\xba\xdc\xfe"sv).BestStringWithErrors(),
+ ERR("unknown token fedcba98"));
+}
+
+alignas(TokenDatabase::RawEntry) constexpr char kDataWithArguments[] =
+ "TOKENS\0\0"
+ "\x09\x00\x00\x00"
+ "\0\0\0\0"
+ "\x00\x00\x00\x00----"
+ "\x0A\x0B\x0C\x0D----"
+ "\x0E\x0F\x00\x01----"
+ "\xAA\xAA\xAA\xAA----"
+ "\xBB\xBB\xBB\xBB----"
+ "\xCC\xCC\xCC\xCC----"
+ "\xDD\xDD\xDD\xDD----"
+ "\xEE\xEE\xEE\xEE----"
+ "\xFF\xFF\xFF\xFF----"
+ "\0"
+ "Use the %s, %s.\0"
+ "Now there are %d of %s!\0"
+ "%c!\0" // AA
+ "%hhu!\0" // BB
+ "%hu!\0" // CC
+ "%u!\0" // DD
+ "%lu!\0" // EE
+ "%llu!"; // FF
+
+constexpr TokenDatabase kWithArgs = TokenDatabase::Create<kDataWithArguments>();
+
+using Case = std::pair<std::string_view, std::string_view>;
+
+template <typename... Args>
+auto TestCases(Args... args) {
+ return std::array<Case, sizeof...(Args)>{args...};
+}
+
+class DetokenizeWithArgs : public ::testing::Test {
+ protected:
+ DetokenizeWithArgs() : detok_(kWithArgs) {}
+
+ Detokenizer detok_;
+};
+
+TEST_F(DetokenizeWithArgs, NoMatches) {
+ EXPECT_TRUE(detok_.Detokenize("\x23\xab\xc9\x87"sv).matches().empty());
+}
+
+TEST_F(DetokenizeWithArgs, SingleMatch) {
+ EXPECT_EQ(detok_.Detokenize("\x00\x00\x00\x00"sv).matches().size(), 1u);
+}
+
+TEST_F(DetokenizeWithArgs, Empty) {
+ EXPECT_EQ(detok_.Detokenize("\x00\x00\x00\x00"sv).BestString(), "");
+}
+
+TEST_F(DetokenizeWithArgs, Successful) {
+ // Run through test cases, but don't include cases that use %hhu or %llu since
+ // these are not currently supported in arm-none-eabi-gcc.
+ for (auto [data, expected] : TestCases(
+ Case{"\x0A\x0B\x0C\x0D\5force\4Luke"sv, "Use the force, Luke."},
+ Case{"\x0E\x0F\x00\x01\4\4them"sv, "Now there are 2 of them!"},
+ Case{"\xAA\xAA\xAA\xAA\xfc\x01"sv, "~!"},
+ Case{"\xCC\xCC\xCC\xCC\xfe\xff\x07"sv, "65535!"},
+ Case{"\xDD\xDD\xDD\xDD\xfe\xff\x07"sv, "65535!"},
+ Case{"\xDD\xDD\xDD\xDD\xfe\xff\xff\xff\x1f"sv, "4294967295!"},
+ Case{"\xEE\xEE\xEE\xEE\xfe\xff\x07"sv, "65535!"},
+ Case{"\xEE\xEE\xEE\xEE\xfe\xff\xff\xff\x1f"sv, "4294967295!"})) {
+ EXPECT_EQ(detok_.Detokenize(data).BestString(), expected);
+ }
+}
+
+TEST_F(DetokenizeWithArgs, ExtraDataError) {
+ auto error = detok_.Detokenize("\x00\x00\x00\x00MORE data"sv);
+ EXPECT_FALSE(error.ok());
+ EXPECT_EQ("", error.BestString());
+}
+
+TEST_F(DetokenizeWithArgs, MissingArgumentError) {
+ auto error = detok_.Detokenize("\x0A\x0B\x0C\x0D\5force"sv);
+ EXPECT_FALSE(error.ok());
+ EXPECT_EQ(error.BestString(), "Use the force, %s.");
+ EXPECT_EQ(error.BestStringWithErrors(),
+ "Use the force, " ERR("%s MISSING") ".");
+}
+
+TEST_F(DetokenizeWithArgs, DecodingError) {
+ auto error = detok_.Detokenize("\x0E\x0F\x00\x01\xFF"sv);
+ EXPECT_FALSE(error.ok());
+ EXPECT_EQ(error.BestString(), "Now there are %d of %s!");
+ EXPECT_EQ(error.BestStringWithErrors(),
+ "Now there are " ERR("%d ERROR") " of " ERR("%s SKIPPED") "!");
+}
+
+alignas(TokenDatabase::RawEntry) constexpr char kDataWithCollisions[] =
+ "TOKENS\0\0"
+ "\x0F\x00\x00\x00"
+ "\0\0\0\0"
+ "\x00\x00\x00\x00\xff\xff\xff\xff" // 1
+ "\x00\x00\x00\x00\x01\x02\x03\x04" // 2
+ "\x00\x00\x00\x00\xff\xff\xff\xff" // 3
+ "\x00\x00\x00\x00\xff\xff\xff\xff" // 4
+ "\x00\x00\x00\x00\xff\xff\xff\xff" // 5
+ "\x00\x00\x00\x00\xff\xff\xff\xff" // 6
+ "\x00\x00\x00\x00\xff\xff\xff\xff" // 7
+ "\xAA\xAA\xAA\xAA\x00\x00\x00\x00" // 8
+ "\xAA\xAA\xAA\xAA\xff\xff\xff\xff" // 9
+ "\xBB\xBB\xBB\xBB\xff\xff\xff\xff" // A
+ "\xBB\xBB\xBB\xBB\xff\xff\xff\xff" // B
+ "\xCC\xCC\xCC\xCC\xff\xff\xff\xff" // C
+ "\xCC\xCC\xCC\xCC\xff\xff\xff\xff" // D
+ "\xDD\xDD\xDD\xDD\xff\xff\xff\xff" // E
+ "\xDD\xDD\xDD\xDD\xff\xff\xff\xff" // F
+ // String table
+ "This string is present\0" // 1
+ "This string is removed\0" // 2
+ "One arg %d\0" // 3
+ "One arg %s\0" // 4
+ "Two args %s %u\0" // 5
+ "Two args %s %s %% %% %%\0" // 6
+ "Four args %d %d %d %d\0" // 7
+ "This one is removed\0" // 8
+ "This one is present\0" // 9
+ "Two ints %d %d\0" // A
+ "Three ints %d %d %d\0" // B
+ "Three strings %s %s %s\0" // C
+ "Two strings %s %s\0" // D
+ "Three %s %s %s\0" // E
+ "Five %d %d %d %d %s\0"; // F
+
+constexpr TokenDatabase kWithCollisions =
+ TokenDatabase::Create<kDataWithCollisions>();
+
+class DetokenizeWithCollisions : public ::testing::Test {
+ protected:
+ DetokenizeWithCollisions() : detok_(kWithCollisions) {}
+
+ Detokenizer detok_;
+};
+
+TEST_F(DetokenizeWithCollisions, Collision_AlwaysPreferSuccessfulDecode) {
+ for (auto [data, expected] :
+ TestCases(Case{"\0\0\0\0"sv, "This string is present"},
+ Case{"\0\0\0\0\x01"sv, "One arg -1"},
+ Case{"\0\0\0\0\x80"sv, "One arg [...]"},
+ Case{"\0\0\0\0\4Hey!\x04"sv, "Two args Hey! 2"})) {
+ EXPECT_EQ(detok_.Detokenize(data).BestString(), expected);
+ }
+}
+
+TEST_F(DetokenizeWithCollisions, Collision_PreferDecodingAllBytes) {
+ for (auto [data, expected] :
+ TestCases(Case{"\0\0\0\0\x80\x80\x80\x80\x00"sv, "Two args [...] 0"},
+ Case{"\0\0\0\0\x08?"sv, "One arg %s"},
+ Case{"\0\0\0\0\x01!\x01\x80"sv, "Two args ! \x80 % % %"})) {
+ EXPECT_EQ(detok_.Detokenize(data).BestString(), expected);
+ }
+}
+
+TEST_F(DetokenizeWithCollisions, Collision_PreferFewestDecodingErrors) {
+ for (auto [data, expected] :
+ TestCases(Case{"\xBB\xBB\xBB\xBB\x00"sv, "Two ints 0 %d"},
+ Case{"\xCC\xCC\xCC\xCC\2Yo\5?"sv, "Two strings Yo %s"})) {
+ EXPECT_EQ(detok_.Detokenize(data).BestString(), expected);
+ }
+}
+
+TEST_F(DetokenizeWithCollisions, Collision_PreferMostDecodedArgs) {
+ auto result = detok_.Detokenize("\xDD\xDD\xDD\xDD\x01\x02\x01\x04\x05"sv);
+ EXPECT_EQ((std::string_view)result.matches()[0].value(), "Five -1 1 -1 2 %s");
+ EXPECT_EQ((std::string_view)result.matches()[1].value(), "Three \2 \4 %s"sv);
+}
+
+TEST_F(DetokenizeWithCollisions, Collision_PreferMostDecodedArgs_NoPercent) {
+ // The "Two args %s %s ..." string successfully decodes this, and has more
+ // "arguments", because of %%, but %% doesn't count as as a decoded argument.
+ EXPECT_EQ(detok_.Detokenize("\0\0\0\0\x01\x00\x01\x02"sv).BestString(),
+ "Four args -1 0 -1 1");
+}
+
+TEST_F(DetokenizeWithCollisions, Collision_PreferStillPresentString) {
+ for (auto [data, expected] :
+ TestCases(Case{"\x00\x00\x00\x00"sv, "This string is present"},
+ Case{"\xAA\xAA\xAA\xAA"sv, "This one is present"})) {
+ EXPECT_EQ(detok_.Detokenize(data).BestString(), expected);
+ }
+}
+
+TEST_F(DetokenizeWithCollisions, Collision_TracksAllMatches) {
+ auto result = detok_.Detokenize("\0\0\0\0"sv);
+ EXPECT_EQ(result.matches().size(), 7u);
+}
+
+} // namespace
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/docs.rst b/pw_tokenizer/docs.rst
new file mode 100644
index 0000000..71abfeb
--- /dev/null
+++ b/pw_tokenizer/docs.rst
@@ -0,0 +1,499 @@
+.. _chapter-tokenizer:
+
+.. default-domain:: cpp
+
+.. highlight:: sh
+
+---------
+Tokenizer
+---------
+The tokenizer module provides facilities for converting strings to binary
+tokens. String literals are replaced with integer tokens in the firmware image,
+which can be decoded off-device to restore the original string. Strings may be
+printf-style format strings and include arguments, such as ``"My name is %s"``.
+Arguments are encoded into compact binary form at runtime.
+
+.. note::
+ This usage of the term "tokenizer" is not related to parsing! The
+ module is called tokenizer because it replaces a whole string literal with an
+ integer token. It does not parse strings into separate tokens.
+
+The most common application of the tokenizer module is binary logging, and its
+designed to integrate easily into existing logging systems. However, the
+tokenizer is general purpose and can be used to tokenize any strings.
+
+**Why tokenize strings?**
+
+ * Dramatically reduce binary size by removing string literals from binaries.
+ * Reduce CPU usage by replacing snprintf calls with simple tokenization code.
+ * Reduce I/O traffic, RAM, and flash usage by sending and storing compact
+ tokens instead of strings.
+ * Remove potentially sensitive log, assert, and other strings from binaries.
+
+Basic operation
+===============
+ 1. In C or C++ code, strings are hashed to generate a stable 32-bit token.
+ 2. The tokenization macro removes the string literal by placing it in an ELF
+ section that is excluded from the final binary.
+ 3. Strings are extracted from an ELF to build a database of tokenized strings
+ for use by the detokenizer. The ELF file may also be used directly.
+ 4. During operation, the device encodes the string token and its arguments, if
+ any.
+ 5. The encoded tokenized strings are sent off-device or stored.
+ 6. Off-device, the detokenizer tools use the token database or ELF files to
+ detokenize the strings to human-readable form.
+
+Module usage
+============
+There are two sides to tokenization: tokenizing strings in the source code and
+detokenizing these strings back to human-readable form.
+
+Tokenization
+------------
+Tokenization converts a string literal to a token. If it's a printf-style
+string, its arguments are encoded along with it. The results of tokenization can
+be sent off device or stored in place of a full string.
+
+Adding tokenization to a project is simple. To tokenize a string, include
+``pw_tokenizer/tokenize.h`` and invoke a ``PW_TOKENIZE_`` macro.
+
+To tokenize a string literal, invoke ``PW_TOKENIZE_STRING``. This macro returns
+a ``uint32_t`` token.
+
+.. code-block:: cpp
+
+ constexpr uint32_t token = PW_TOKENIZE_STRING("Any string literal!");
+
+Format strings are tokenized into a fixed-size buffer. The buffer contains the
+``uint32_t`` token followed by the encoded form of the arguments, if any. The
+most flexible tokenization macro is ``PW_TOKENIZE_TO_BUFFER``, which encodes to
+a caller-provided buffer.
+
+.. code-block:: cpp
+
+ uint8_t buffer[BUFFER_SIZE];
+ size_t size_bytes = sizeof(buffer);
+ PW_TOKENIZE_TO_BUFFER(buffer, &size_bytes, format_string_literal, args...);
+
+While ``PW_TOKENIZE_TO_BUFFER`` is flexible, its per-use code size overhead is
+larger than its alternatives. ``PW_TOKENIZE_TO_CALLBACK`` tokenizes to a buffer
+on the stack and calls a ``void(const uint8_t* buffer, size_t buffer_size)``
+callback that is provided at the call site. The size of the buffer is set with
+``PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES``.
+
+.. code-block:: cpp
+
+ PW_TOKENIZE_TO_CALLBACK(HandlerFunction, "Format string: %x", arg);
+
+``PW_TOKENIZE_TO_GLOBAL_HANDLER`` is the most efficient tokenization function,
+since it takes the fewest arguments. Like the callback form, it encodes to a
+buffer on the stack. It then calls the C-linkage function
+``pw_TokenizerHandleEncodedMessage``, which must be defined by the project.
+
+.. code-block:: cpp
+
+ PW_TOKENIZE_TO_GLOBAL_HANDLER(format_string_literal, arguments...);
+
+ void pw_TokenizerHandleEncodedMessage(const uint8_t encoded_message[],
+ size_t size_bytes);
+
+``PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD`` is similar, but passes a
+``void*`` argument to the global handler function. This can be used to pass a
+log level or other metadata along with the tokenized string.
+
+.. code-block:: cpp
+
+ PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD(payload,
+ format_string_literal,
+ args...);
+
+ void pw_TokenizerHandleEncodedMessageWithPayload(void* payload,
+ const uint8_t encoded_message[],
+ size_t size_bytes);
+
+.. tip::
+ ``%s`` arguments are inefficient to encode and can quickly fill a tokenization
+ buffer. Keep ``%s`` arguments short or avoid encoding them as strings if
+ possible. See `Tokenized strings as %s arguments`_.
+
+Example: binary logging
+^^^^^^^^^^^^^^^^^^^^^^^
+String tokenization is perfect for logging. Consider the following log macro,
+which gathers the file, line number, and log message. It calls the ``RecordLog``
+function, which formats the log string, collects a timestamp, and transmits the
+result.
+
+.. code-block:: cpp
+
+ #define LOG_INFO(format, ...) \
+ RecordLog(LogLevel_INFO, __FILE_NAME__, __LINE__, format, ##__VA_ARGS__)
+
+ void RecordLog(LogLevel level, const char* file, int line, const char* format,
+ ...) {
+ if (level < current_log_level) {
+ return;
+ }
+
+ int bytes = snprintf(buffer, sizeof(buffer), "%s:%d ", file, line);
+
+ va_list args;
+ va_start(args, format);
+ bytes += vsnprintf(&buffer[bytes], sizeof(buffer) - bytes, format, args);
+ va_end(args);
+
+ TransmitLog(TimeSinceBootMillis(), buffer, size);
+ }
+
+It is trivial to convert this to a binary log using the tokenizer. The
+``RecordLog`` call is replaced with a
+``PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD`` invocation. The
+``pw_TokenizerHandleEncodedMessageWithPayload`` implementation collects the
+timestamp and transmits the message with ``TransmitLog``.
+
+.. code-block:: cpp
+
+ #define LOG_INFO(format, ...) \
+ PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD( \
+ (void*)LogLevel_INFO, \
+ __FILE_NAME__ ":%d " format, \
+ __LINE__, \
+ __VA_ARGS__); \
+
+ extern "C" void pw_TokenizerHandleEncodedMessageWithPayload(
+ void* level, const uint8_t encoded_message[], size_t size_bytes) {
+ if (reinterpret_cast<LogLevel>(level) >= current_log_level) {
+ TransmitLog(TimeSinceBootMillis(), encoded_message, size_bytes);
+ }
+ }
+
+Note that the ``__FILE_NAME__`` string is directly included in the log format
+string. Since the string is tokenized, this has no effect on binary size. A
+``%d`` for the line number is added to the format string, so that changing the
+line of the log message does not generate a new token. There is no overhead for
+additional tokens, but it may not be desireable to fill a token database with
+duplicate log lines.
+
+Database management
+^^^^^^^^^^^^^^^^^^^
+Token databases store a mapping of tokens to the strings they represent. An ELF
+file can be used as a token database, but it only contains the strings for its
+exact build. A token database file aggregates tokens from multiple ELF files, so
+that a single database can decode tokenized strings from any known ELF.
+
+Creating and maintaining a token database is simple. Token databases are managed
+with the ``database.py`` script. The ``create`` command makes a new token
+database from ELF files or other databases.
+
+.. code-block:: sh
+
+ ./database.py create --database DATABASE_NAME ELF_OR_DATABASE_FILE...
+
+Two database formats are supported: CSV and binary. Provide ``--type binary`` to
+``create`` generate a binary database instead of the default CSV. CSV databases
+are great for checking into a source control or for human review. Binary
+databases are more compact and simpler to parse. The C++ detokenizer library
+only supports binary databases currently.
+
+As new tokenized strings are added, update the database with the add command.
+
+.. code-block:: sh
+
+ ./database.py add --database DATABASE_NAME ELF_OR_DATABASE_FILE...
+
+A CSV token database can be checked into a source repository and updated as code
+changes are made. The build system can invoke ``database.py`` to update the
+database after each build.
+
+Detokenization
+--------------
+Detokenization is the process of expanding a token to the string it represents
+and decoding its arguments. This module provides Python and C++ detokenization
+libraries.
+
+Python
+^^^^^^
+To detokenize in Python, import Detokenizer from the ``pw_tokenizer`` package,
+and instantiate it with paths to token databases or ELF files.
+
+.. code-block:: python
+
+ import pw_tokenizer
+
+ detokenizer = pw_tokenizer.Detokenizer('path/to/database.csv', 'other/path.elf')
+
+ def process_log_message(log_message):
+ result = detokenizer.detokenize(log_message.payload)
+ self._log(str(result))
+
+The ``pw_tokenizer`` pacakge also provices the ``AutoUpdatingDetokenizer``
+class, which can be used in place of the standard ``Detokenizer``. This class
+monitors database files for changes and automatically reloads them when they
+change. This is helpful for long-running tools that use detokenization.
+
+C++
+^^^
+The C++ detokenization libraries can be used in C++ or any language that can
+call into C++ with a C-linkage wrapper, such as Java or Rust. A reference
+Android Java JNI is provided.
+
+The C++ detokenization library uses binary-format token databases (created with
+``--type binary``). Read a binary format database from a file or include it in
+the source code. Pass the database array to ``TokenDatabase::Create``, and
+construct a detokenizer.
+
+.. code-block:: cpp
+
+ Detokenizer detokenizer(TokenDatabase::Create(token_database_array));
+
+ std::string ProcessLog(span<uint8_t> log_data) {
+ return detokenizer.Detokenize(log_data).BestString();
+ }
+
+The ``TokenDatabase`` class verifies that its data is valid before using it. If
+it is invalid, the ``TokenDatabase::Create`` returns an empty database for which
+``ok()`` returns false. If the token database is included in the source code,
+this check can be done at compile time.
+
+.. code-block:: cpp
+
+ // This line fails to compile with a static_assert if the database is invalid.
+ constexpr TokenDatabase kDefaultDatabase = TokenDatabase::Create<kData>();
+
+ Detokenizer OpenDatabase(std::string_view path) {
+ std::vector<uint8_t> data = ReadWholeFile(path);
+
+ TokenDatabase database = TokenDatabase::Create(data);
+
+ // This checks if the file contained a valid database. It is safe to use a
+ // TokenDatabase that failed to load (it will be empty), but it may be
+ // desirable to provide a default database or otherwise handle the error.
+ if (database.ok()) {
+ return Detokenizer(database);
+ }
+ return Detokenizer(kDefaultDatabase);
+ }
+
+Token generation: fixed length hashing at compile time
+======================================================
+String tokens are generated using a modified version of the x65599 hash used by
+the SDBM project. All hashing is done at compile time.
+
+In C code, strings are hashed with a preprocessor macro. For compatibility with
+macros, the hash must be limited to a fixed maximum number of characters. This
+value is set by ``PW_TOKENIZER_CFG_HASH_LENGTH``.
+
+Increasing ``PW_TOKENIZER_CFG_HASH_LENGTH`` increases the compilation time for C
+due to the complexity of the hashing macros. C++ macros use a constexpr
+function instead of a macro, so the compilation time impact is minimal. Projects
+primarily in C++ should use a large value for ``PW_TOKENIZER_CFG_HASH_LENGTH``
+(perhaps even ``std::numeric_limits<size_t>::max()``).
+
+Base64 format
+=============
+The tokenizer defaults to a compact binary representation of tokenized messages.
+Applications may desire a textual representation of tokenized strings. This
+makes it easy to use tokenized messages alongside plain text messages, but comes
+at an efficiency cost.
+
+The tokenizer module supports prefixed Base64-encoded messages: a single
+character ($) followed by the Base64-encoded message. For example, the token
+0xabcdef01 followed by the argument 0x05 would be encoded as ``01 ef cd ab 05``
+in binary and ``$Ae/NqwU=`` in Base64.
+
+Base64 decoding is supported in the Python detokenizer through the
+``detokenize_base64`` and related functions. Base64 encoding and decoding are
+not yet supprted in C++, but it is straightforward to add Base64 encoding with
+any Base64 library.
+
+.. tip::
+ The detokenization tools support recursive detokenization for prefixed Base64
+ text. Tokenized strings found in detokenized text are detokenized, so
+ prefixed Base64 messages can be passed as ``%s`` arguments.
+
+ For example, the message ``"$d4ffJaRn`` might be passed as the argument to a
+ ``"Nested message: %s"`` string. The detokenizer would decode the message in
+ two steps:
+
+ ::
+
+ "$alRZyuk2J3v=" → "Nested message: $d4ffJaRn" → "Nested message: Wow!"
+
+War story: deploying tokenized logging to an existing product
+=============================================================
+The tokenizer module was developed to bring tokenized logging to an
+in-development product. The product is complex, with several interacting
+microcontrollers. It already had an established text-based logging system.
+Deploying tokenization was straightforward and had substantial benefits.
+
+**Results**
+ * Log contents shrunk by over 50%, even with Base64 encoding.
+
+ * Significant size savings for encoded logs, even using the less-efficient
+ Base64 encoding required for compatibility with the existing log system.
+ * Freed valueable communication bandwidth.
+ * Allowed storing many more logs in crash dumps.
+
+ * Substantial flash savings.
+
+ * Reduced the size of 115 KB and 172 KB firmware images by over 20 KB each.
+ * Shaved over 100 KB from a large 2 MB image.
+
+ * Simpler logging code.
+
+ * Removed CPU-heavy ``snprintf`` calls.
+ * Removed complex code for forwarding log arguments to a low-priority task.
+
+This section describes the tokenizer deployment process and highlights key
+insights.
+
+Firmware deployment
+-------------------
+ * In the project's logging macro, calls to the underlying logging function
+ were replaced with a ``PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD``
+ invocation.
+ * The log level was passed as the payload argument to facilitate runtime log
+ level control.
+ * For this project, it was necessary to encode the log messages as text. In
+ ``pw_TokenizerHandleEncodedMessageWithPayload``, the log messages were
+ encoded in the $-prefixed `Base64 format`_, then dispatched as normal log
+ messages.
+ * Asserts were tokenized using ``PW_TOKENIZE_TO_CALLBACK``.
+
+.. attention::
+ Do not encode line numbers in tokenized strings. This results in a huge
+ number of lines being added to the database, since every time code moves,
+ new strings are tokenized. If line numbers are desired in a tokenized
+ string, add a ``"%d"`` to the string and pass ``__LINE__`` as an argument.
+
+Database management
+-------------------
+ * The token database was stored as a CSV file in the project's Git repo.
+ * The token database was automatically updated as part of the build, and
+ developers were expected to check in the database changes alongside their
+ code changes.
+ * A presubmit check verified that all strings added by a change were added to
+ the token database.
+ * The token database included logs and asserts for all firmware images in the
+ project.
+ * No strings were purged from the token database.
+
+.. tip::
+ Merge conflicts may be a frequent occurrence with an in-source database. If
+ the database is in-source, make sure there is a simple script to resolve any
+ merge conflicts. The script could either keep both sets of lines or discard
+ local changes and regenerate the database.
+
+Decoding tooling deployment
+---------------------------
+ * The Python detokenizer in ``pw_tokenizer`` was deployed to two places:
+
+ * Product-specific Python command line tools, using
+ ``pw_tokenizer.Detokenizer``.
+ * Standalone script for decoding prefixed Base64 tokens in files or
+ live output (e.g. from ``adb``), using ``detokenize.py``'s command line
+ interface.
+
+ * The C++ detokenizer library was deployed to two Android apps with a Java
+ Native Interface (JNI) layer.
+
+ * The binary token database was included as a raw resource in the APK.
+ * In one app, the built-in token database could be overridden by copying a
+ file to the phone.
+
+.. tip:: Make the tokenized logging tools simple to use.
+
+ * Provide simple wrapper shell scripts that fill in arguments for the
+ project. For example, point ``detokenize.py`` to the project's token
+ databases.
+ * Use ``pw_tokenizer.AutoReloadingDetokenizer`` to decode in
+ continuously-running tools, so that users don't have to restart the tool
+ when the token database updates.
+ * Integrate detokenization everywhere it is needed. Integrating the tools
+ takes just a few lines of code, and token databases can be embedded in
+ APKs or binaries.
+
+Limitations and future work
+===========================
+
+GCC bug: tokenization in template functions
+-------------------------------------------
+GCC incorrectly ignores the section attribute for template
+`functions <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=70435>`_ and
+`variables <https://gcc.gnu.org/bugzilla/show_bug.cgi?id=88061>`_. Due to this
+bug, tokenized strings in template functions may be emitted into ``.rodata``
+instead of the special tokenized string section. This causes two problems:
+
+ 1. Tokenized strings will not be discovered by the token database tools.
+ 2. Tokenized strings may not be removed from the final binary.
+
+clang does **not** have this issue! Use clang if you can.
+
+It is possible to work around this bug in GCC. One approach would be to tag
+format strings so that the database tools can find them in ``.rodata``. Then, to
+remove the strings, compile two binaries: one metadata binary with all tokenized
+strings and a second, final binary that removes the strings. The strings could
+be removed by providing the appropriate linker flags or by removing the ``used``
+attribute from the tokenized string character array declaration.
+
+64-bit tokenization
+-------------------
+The Python and C++ detokenizing libraries currently assume that strings were
+tokenized on a system with 32-bit ``long``, ``size_t``, ``intptr_t``, and
+``ptrdiff_t``. Decoding may not work correctly for these types if a 64-bit
+device performed the tokenization.
+
+Supporting detokenization of strings tokenized on 64-bit targets would be
+simple. This could be done by adding an option to switch the 32-bit types to
+64-bit. The tokenizer stores the sizes of these types in the ``.tokenizer_info``
+ELF section, so the sizes of these types can be verified by checking the ELF
+file, if necessary.
+
+Tokenization in headers
+-----------------------
+Tokenizing code in header files (inline functions or templates) may trigger
+warnings such as ``-Wlto-type-mismatch`` under certain conditions. That
+is because tokenization requires declaring a character array for each tokenized
+string. If the tokenized string includes macros that change value, the size of
+this character array changes, which means the same static variable is defined
+with different sizes. It should be safe to suppress these warnings, but, when
+possible, code that tokenizes strings with macros that can change value should
+be moved to source files rather than headers.
+
+Tokenized strings as ``%s`` arguments
+-------------------------------------
+Encoding ``%s`` string arguments is inefficient, since ``%s`` strings are
+encoded 1:1, with no tokenization. It would be better to send a tokenized string
+literal as an integer instead of a string argument, but this is not yet
+supported.
+
+A string token could be sent by marking an integer % argument in a way
+recognized by the detokenization tools. The detokenizer would expand the
+argument to the string represented by the integer.
+
+.. code-block:: cpp
+
+ #define PW_TOKEN_ARG "TOKEN<([\\%" PRIx32 "/])>END_TOKEN"
+
+ constexpr uint32_t answer_token = PW_TOKENIZE_STRING("Uh, who is there");
+
+ PW_TOKENIZE_TO_GLOBAL_HANDLER("Knock knock: " PW_TOKEN_ARG "?", answer_token);
+
+Strings with arguments could be encoded to a buffer, but since printf strings
+are null-terminated, a binary encoding would not work. These strings can be
+prefixed Base64-encoded and sent as ``%s`` instead. See `Base64 format`_.
+
+Another possibility: encode strings with arguments to a ``uint64_t`` and send
+them as an integer. This would be efficient and simple, but only support a small
+number of arguments.
+
+Compatibility
+=============
+ * C11
+ * C++17
+ * Python 3
+
+Dependencies
+============
+ * pw_varint module
+ * pw_preprocessor module
+ * pw_span module
diff --git a/pw_tokenizer/generate_decoding_test_data.cc b/pw_tokenizer/generate_decoding_test_data.cc
new file mode 100644
index 0000000..bc9c595
--- /dev/null
+++ b/pw_tokenizer/generate_decoding_test_data.cc
@@ -0,0 +1,471 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// This program generates Python test data for decoder_test.py.
+//
+// To generate the test data, build the target
+// pw_tokenizer_generate_decoding_test_data. Execute the binary and move the
+// generated files to this directory.
+
+#include <array>
+#include <cctype>
+#include <cinttypes>
+#include <cstdarg>
+#include <cstdint>
+#include <cstdio>
+#include <random>
+
+#include "pw_span/span.h"
+#include "pw_tokenizer/internal/decode.h"
+#include "pw_tokenizer/tokenize.h"
+#include "pw_varint/varint.h"
+
+namespace {
+
+// Defines how to format test cases for the target language.
+struct SourceFileFormat {
+ const char* extension;
+ const char* comment;
+ const char* header;
+ const char* footer;
+ const char* test_case_prefix;
+ const char* binary_string_prefix;
+ const char* binary_string_suffix;
+};
+
+// clang-format off
+constexpr const char* kCopyrightLines[] = {
+"Copyright 2020 The Pigweed Authors",
+"",
+"Licensed under the Apache License, Version 2.0 (the \"License\"); you may not",
+"use this file except in compliance with the License. You may obtain a copy of",
+"the License at",
+"",
+" https://www.apache.org/licenses/LICENSE-2.0",
+"",
+"Unless required by applicable law or agreed to in writing, software",
+"distributed under the License is distributed on an \"AS IS\" BASIS, WITHOUT",
+"WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the",
+"License for the specific language governing permissions and limitations under",
+"the License.",
+};
+// clang-format on
+
+// The header includes a %s for the name and a %s for the test case type.
+constexpr const char kCcHeader[] = R"(#pragma once
+
+#include <string_view>
+#include <tuple>
+
+namespace pw::test::%s {
+
+using namespace std::literals::string_view_literals;
+
+// clang-format off
+using TestCase = %s;
+
+inline constexpr TestCase kTestData[] = {
+)";
+
+constexpr const char kCcFooter[] = R"(
+};
+
+} // namespace pw::test::%s
+)";
+
+constexpr const char kPythonHeader[] = R"("""Generated test data."""
+
+# pylint: disable=line-too-long
+# C++ test case type for %s:
+# %s
+
+
+def TestCase(*args): # pylint: disable=invalid-name
+ return tuple(args)
+
+
+# yapf: disable
+TEST_DATA = (
+)";
+
+constexpr SourceFileFormat kCcFormat{
+ ".h", "//", kCcHeader, kCcFooter, "TestCase", "\"", "\"sv"};
+
+constexpr SourceFileFormat kPythonFormat{
+ ".py", "#", kPythonHeader, "\n)\n", "", "b'", "'"};
+
+class TestDataFile {
+ public:
+ TestDataFile(const char* name,
+ const SourceFileFormat& format,
+ const char* test_case_format)
+ : format_(format),
+ name_(name),
+ test_case_format_(test_case_format),
+ path_(std::string(name) + "_test_data" + format_.extension),
+ file_(std::fopen(path_.c_str(), "w")) {}
+
+ ~TestDataFile() { std::fclose(file_); }
+
+ const SourceFileFormat& fmt() const { return format_; }
+ const std::string& path() const { return path_; }
+
+ // Writes a file with test cases uses the provided function.
+ void WriteTestCases(void (*function)(TestDataFile*)) {
+ static constexpr const char* kFileBase =
+ &__FILE__[std::string_view(__FILE__).find_last_of('/') + 1];
+
+ for (const char* line : kCopyrightLines) {
+ printf("%s", fmt().comment);
+ if (line[0] == '\0') {
+ printf("\n");
+ } else {
+ printf(" %s\n", line);
+ }
+ }
+
+ printf("\n%s AUTOGENERATED - DO NOT EDIT\n", fmt().comment);
+ printf("%s This file contains test data generated by %s.\n",
+ fmt().comment,
+ kFileBase);
+
+ printf(fmt().header, name_, test_case_format_);
+ function(this);
+ printf(fmt().footer, name_);
+ }
+
+ // Starts a section of test cases in the file.
+ void Section(const char* comment) {
+ printf("\n%s %s\n", fmt().comment, comment);
+ }
+
+ int printf(const char* format, ...) PW_PRINTF_FORMAT(2, 3) {
+ va_list args;
+ va_start(args, format);
+ const int result = std::vfprintf(file_, format, args);
+ va_end(args);
+ return result;
+ }
+
+ private:
+ SourceFileFormat format_;
+ const char* name_;
+ const char* test_case_format_;
+ std::string path_;
+ FILE* file_;
+};
+
+// Writes a decoding test case to the file.
+void TestCase(TestDataFile* file,
+ pw::span<const uint8_t> buffer,
+ const char* format,
+ const char* formatted) {
+ file->printf(R"(TestCase("%s", "%s", %s)",
+ format,
+ formatted,
+ file->fmt().binary_string_prefix);
+
+ for (uint8_t byte : buffer) {
+ file->printf("\\x%02x", byte);
+ }
+
+ file->printf("%s),\n", file->fmt().binary_string_suffix);
+}
+
+template <size_t kSize>
+void TestCase(TestDataFile* file,
+ const char* format,
+ const char (&buffer)[kSize],
+ const char* formatted) {
+ TestCase(file,
+ pw::span(reinterpret_cast<const uint8_t*>(buffer), kSize - 1),
+ format,
+ formatted);
+}
+
+// __VA_ARGS__ is expanded twice, so ONLY variables / constants should be used.
+#define MAKE_TEST_CASE(format, ...) \
+ do { \
+ std::array<uint8_t, 128> buffer; \
+ size_t size = buffer.size(); \
+ PW_TOKENIZE_TO_BUFFER(buffer.data(), &size, format, ##__VA_ARGS__); \
+ \
+ std::array<char, 128> formatted = {}; \
+ std::snprintf(formatted.data(), formatted.size(), format, ##__VA_ARGS__); \
+ TestCase(file, \
+ pw::span(buffer).first(size).subspan(4), /* skip the token */ \
+ format, \
+ formatted.data()); \
+ } while (0)
+
+// Formats the contents like an error.
+#define ERROR_STR PW_TOKENIZER_ARG_DECODING_ERROR
+
+extern "C" void pw_TokenizerHandleEncodedMessage(const uint8_t*, size_t) {}
+extern "C" void pw_TokenizerHandleEncodedMessageWithPayload(pw_TokenizerPayload,
+ const uint8_t*,
+ size_t) {}
+
+// Generates data to test tokenized string decoding.
+void GenerateEncodedStrings(TestDataFile* file) {
+ std::mt19937 random(6006411);
+ std::uniform_int_distribution<int64_t> big;
+ std::uniform_int_distribution<int32_t> medium;
+ std::uniform_int_distribution<char> small(' ', '~');
+ std::uniform_real_distribution<float> real;
+
+ file->Section("Simple strings");
+ TestCase(file, "%s", "\3SFO", "SFO");
+ TestCase(file, "%s", "\4KSJC", "KSJC");
+ TestCase(file, "%s", "\0", "");
+
+ TestCase(file, "%5s%s", "\2no\3fun", " nofun");
+ TestCase(file, "%5s%s", "\6abcdef\0", "abcdef");
+ TestCase(file, "%5s%s", "\0\6abcdef", " abcdef");
+
+ TestCase(file,
+ "%s %-6s%s%s%s",
+ "\5Intel\580586\7toaster\1 \4oven",
+ "Intel 80586 toaster oven");
+ TestCase(file,
+ "%s %-6s%s%s%s",
+ "\5Apple\x09"
+ "automatic\7 pencil\1 \x09sharpener",
+ "Apple automatic pencil sharpener");
+
+ file->Section("Zero-length strings");
+ TestCase(file, "%s-%s", "\x02so\x00", "so-");
+ TestCase(file, "%s-%s", "\x00\04cool", "-cool");
+ TestCase(file, "%s%s%3s%s", "\0\0\0\0", " ");
+ TestCase(file, "(%5s)(%2s)(%7s)", "\x80\0\x80", "([...])( )( [...])");
+
+ file->Section("Invalid strings");
+ TestCase(file, "%s", "\x03hi", ERROR_STR("%s ERROR (hi)"));
+ TestCase(file, "%30s", "\x03hi", ERROR_STR("%30s ERROR (hi)"));
+ TestCase(file, "%30s", "\x83hi", ERROR_STR("%30s ERROR (hi)"));
+ TestCase(file, "%s", "\x85yo!", ERROR_STR("%s ERROR (yo!)"));
+ TestCase(file, "%s", "\x01", ERROR_STR("%s ERROR"));
+ TestCase(file, "%30s", "\x81", ERROR_STR("%30s ERROR"));
+
+ file->Section("Continue after truncated string");
+ TestCase(file, "%s %d %s", "\x82go\4\5lunch", "go[...] 2 lunch");
+ TestCase(file, "%6s%s%s", "\x80\x85hello\x05there", " [...]hello[...]there");
+
+ file->Section("Floating point");
+ TestCase(file, "%1.1f", "\0\0\0\0", "0.0");
+ TestCase(file, "%0.5f", "\xdb\x0f\x49\x40", "3.14159");
+
+ file->Section("Character"); // ZigZag doubles the value of positive integers.
+ TestCase(file, "%c", "\x40", " "); // 0x20
+ TestCase(file, "%c", "\x48", "$"); // 0x24
+ TestCase(file, "%c", "\x48", "$"); // 0x24
+ TestCase(file, "100%c!", "\x4A", "100%!"); // 0x25
+
+ file->Section("Atypical argument types");
+ MAKE_TEST_CASE("%ju", static_cast<uintmax_t>(99));
+ MAKE_TEST_CASE("%jd", static_cast<intmax_t>(99));
+ MAKE_TEST_CASE("%zu", sizeof(uint64_t));
+ MAKE_TEST_CASE("%zd", static_cast<ssize_t>(123));
+ MAKE_TEST_CASE("%td", static_cast<ptrdiff_t>(99));
+
+ file->Section("Percent character");
+ TestCase(file, "%%", "", "%");
+ TestCase(file, "%%%%%%%%", "abc", "%%%%");
+ TestCase(file, "whoa%%%%wow%%%%!%%", "", "whoa%%wow%%!%");
+ TestCase(file, "This is %d%% effective", "\x02", "This is 1% effective");
+ TestCase(
+ file, "%% is 100%sa%%sign%%%s", "\x01%\x03OK?", "% is 100%a%sign%OK?");
+
+ file->Section("Percent character prints after errors");
+ TestCase(file, "%s%%", "\x83-10\0", "-10[...]%");
+ TestCase(
+ file, "%d%% is a good %%", "", ERROR_STR("%d MISSING") "% is a good %");
+
+ file->Section("Various format strings");
+ MAKE_TEST_CASE("!");
+ MAKE_TEST_CASE("%s", "%s");
+ MAKE_TEST_CASE("%s", "hello");
+ MAKE_TEST_CASE("%s%s", "Hello", "old");
+ MAKE_TEST_CASE("%s to the%c%s", "hello", ' ', "whirled");
+ MAKE_TEST_CASE("hello %s %d %d %d", "rolled", 1, 2, 3);
+
+ TestCase(file, "", "", "");
+ TestCase(file, "This has no specifiers", "", "This has no specifiers");
+ TestCase(file, "%s_or_%3s", "\x05hello\x02hi", "hello_or_ hi");
+ TestCase(file, "%s_or_%3d", "\x05hello\x7f", "hello_or_-64");
+ TestCase(file,
+ "%s or hi%c pi=%1.2e",
+ "\x05hello\x42\xdb\x0f\x49\x40",
+ "hello or hi! pi=3.14e+00");
+ TestCase(file,
+ "Why, %s there. My favorite number is %.2f%c",
+ "\x05hello\xdb\x0f\x49\x40\x42",
+ "Why, hello there. My favorite number is 3.14!");
+
+ file->Section("Various errors");
+ TestCase(file, "%d", "", ERROR_STR("%d MISSING"));
+
+ TestCase(file,
+ "ABC%d123%dabc%dABC",
+ "",
+ "ABC" ERROR_STR("%d MISSING") "123" ERROR_STR(
+ "%d SKIPPED") "abc" ERROR_STR("%d SKIPPED") "ABC");
+
+ TestCase(file,
+ "%sXY%+ldxy%a",
+ "\x83Yo!\x80",
+ "Yo![...]XY" ERROR_STR("%+ld ERROR") "xy" ERROR_STR("%a SKIPPED"));
+
+ TestCase(file, "%d", "", ERROR_STR("%d MISSING"));
+
+ TestCase(file,
+ "%sXY%+ldxy%a",
+ "\x83Yo!\x80",
+ "Yo![...]XY" ERROR_STR("%+ld ERROR") "xy" ERROR_STR("%a SKIPPED"));
+
+ TestCase(file,
+ "%s%lld%9u",
+ "\x81$\x80\x80",
+ "$[...]" ERROR_STR("%lld ERROR") ERROR_STR("%9u SKIPPED"));
+
+ file->Section("Alternate form (#)");
+ MAKE_TEST_CASE("Hex: %#x", 0xbeef);
+ MAKE_TEST_CASE("Hex: %#08X", 0xfeed);
+
+ file->Section("Random integers");
+ for (int i = 0; i < 100; ++i) {
+ float f = real(random);
+ MAKE_TEST_CASE(
+ "This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", f, f, i, f, f, f, f);
+ }
+
+ for (int i = 0; i < 100; ++i) {
+ unsigned long long n1 = big(random);
+ int n2 = medium(random);
+ char ch = small(random);
+ if (ch == '"' || ch == '\\') {
+ ch = '\t';
+ }
+
+ MAKE_TEST_CASE("%s: %llu %d %c", std::to_string(i).c_str(), n1, n2, ch);
+ }
+
+ for (int i = 0; i < 100; ++i) {
+ const long long n1 = big(random);
+ const unsigned n2 = medium(random);
+ const char ch = small(random);
+
+ MAKE_TEST_CASE(
+ "%s: %lld 0x%16u%08X %d", std::to_string(i).c_str(), n1, n2, n2, ch);
+ }
+}
+
+template <typename T>
+void OutputVarintTest(TestDataFile* file, T i) {
+ if constexpr (sizeof(T) <= sizeof(int)) {
+ file->printf(R"(TestCase("%%d", "%d", "%%u", "%u", %s)",
+ static_cast<int>(i),
+ static_cast<unsigned>(i),
+ file->fmt().binary_string_prefix);
+ } else {
+ file->printf(R"(TestCase("%%lld", "%lld", "%%llu", "%llu", %s)",
+ static_cast<long long>(i),
+ static_cast<unsigned long long>(i),
+ file->fmt().binary_string_prefix);
+ }
+
+ std::array<uint8_t, 10> buffer;
+ // All integers are encoded as signed for tokenization.
+ size_t size = pw::varint::Encode(i, pw::as_writable_bytes(pw::span(buffer)));
+
+ for (size_t i = 0; i < size; ++i) {
+ file->printf("\\x%02x", buffer[i]);
+ }
+
+ file->printf("%s),\n", file->fmt().binary_string_suffix);
+}
+
+// Generates data to test variable-length integer decoding.
+void GenerateVarints(TestDataFile* file) {
+ std::mt19937 random(6006411);
+ std::uniform_int_distribution<int64_t> signed64;
+ std::uniform_int_distribution<int32_t> signed32;
+ std::uniform_int_distribution<int16_t> signed16;
+
+ file->Section("Important numbers");
+ OutputVarintTest(file, 0);
+ OutputVarintTest(file, std::numeric_limits<int16_t>::min());
+ OutputVarintTest(file, std::numeric_limits<int16_t>::min() + 1);
+ OutputVarintTest(file, std::numeric_limits<int16_t>::max() - 1);
+ OutputVarintTest(file, std::numeric_limits<int16_t>::max());
+ OutputVarintTest(file, std::numeric_limits<int32_t>::min());
+ OutputVarintTest(file, std::numeric_limits<int32_t>::min() + 1);
+ OutputVarintTest(file, std::numeric_limits<int32_t>::max() - 1);
+ OutputVarintTest(file, std::numeric_limits<int32_t>::max());
+ OutputVarintTest(file, std::numeric_limits<int64_t>::min());
+ OutputVarintTest(file, std::numeric_limits<int64_t>::min() + 1);
+ OutputVarintTest(file, std::numeric_limits<int64_t>::max() - 1);
+ OutputVarintTest(file, std::numeric_limits<int64_t>::max());
+
+ file->Section("Random 64-bit ints");
+ for (int i = 0; i < 500; ++i) {
+ OutputVarintTest(file, signed64(random));
+ }
+ file->Section("Random 32-bit ints");
+ for (int i = 0; i < 100; ++i) {
+ OutputVarintTest(file, signed32(random));
+ }
+ file->Section("Random 16-bit ints");
+ for (int i = 0; i < 100; ++i) {
+ OutputVarintTest(file, signed16(random));
+ }
+
+ file->Section("All 8-bit numbers");
+ {
+ int i = std::numeric_limits<int8_t>::min();
+ while (true) {
+ OutputVarintTest(file, i);
+ if (i == std::numeric_limits<int8_t>::max()) {
+ break;
+ }
+ // Don't use an inline increment to avoid undefined behavior (overflow).
+ i += 1;
+ }
+ }
+}
+
+template <typename Function>
+void WriteFile(const char* name,
+ const char* test_case_format,
+ Function function) {
+ for (const SourceFileFormat& file_format : {kCcFormat, kPythonFormat}) {
+ TestDataFile file(name, file_format, test_case_format);
+ file.WriteTestCases(function);
+
+ std::printf("Wrote %s\n", file.path().c_str());
+ }
+}
+
+} // namespace
+
+int main(int, char**) {
+ WriteFile("tokenized_string_decoding",
+ "std::tuple<const char*, std::string_view, std::string_view>",
+ GenerateEncodedStrings);
+ WriteFile("varint_decoding",
+ "std::tuple<const char*, const char*, const char*, const char*, "
+ "std::string_view>",
+ GenerateVarints);
+ return 0;
+}
diff --git a/pw_tokenizer/hash_test.cc b/pw_tokenizer/hash_test.cc
new file mode 100644
index 0000000..8392437
--- /dev/null
+++ b/pw_tokenizer/hash_test.cc
@@ -0,0 +1,240 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include <cstddef>
+#include <cstdint>
+
+#include "gtest/gtest.h"
+#include "pw_preprocessor/util.h"
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h"
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h"
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h"
+#include "pw_tokenizer/pw_tokenizer_65599_fixed_length_hash.h"
+#include "pw_tokenizer_private/generated_hash_test_cases.h"
+
+namespace pw::tokenizer {
+namespace {
+
+// Make sure the array test cases file isn't empty.
+static_assert(PW_ARRAY_SIZE(kHashTests) > 10,
+ "The test cases array is suspiciously small");
+
+constexpr bool CheckGeneratedCases() {
+ for (const auto [string, hash_length, python_hash, macro_hash] : kHashTests) {
+ const uint32_t calculated_hash =
+ PwTokenizer65599FixedLengthHash(string, hash_length);
+ if (calculated_hash != python_hash || calculated_hash != macro_hash) {
+ return false;
+ }
+ }
+
+ return true;
+}
+
+static_assert(CheckGeneratedCases(),
+ "Hashes in the generated test cases must match");
+
+TEST(Hashing, GeneratedCasesAtRuntime) {
+ for (const auto [string, hash_length, python_hash, macro_hash] : kHashTests) {
+ const uint32_t calculated_hash =
+ PwTokenizer65599FixedLengthHash(string, hash_length);
+ EXPECT_EQ(calculated_hash, python_hash);
+ EXPECT_EQ(calculated_hash, macro_hash);
+ }
+}
+
+// Gets the size of the string, excluding the null terminator. A uint32_t is
+// used instead of a size_t since the hash calculation requires a uint32_t.
+template <uint32_t kSizeIncludingNull>
+constexpr uint32_t StringLength(const char (&)[kSizeIncludingNull]) {
+ static_assert(kSizeIncludingNull > 0u);
+ return kSizeIncludingNull - 1; // subtract the null terminator
+}
+
+TEST(Hashing, Runtime) {
+ // Coefficients for the hash terms; k1 is 1.
+ static constexpr uint32_t k2 = k65599HashConstant;
+ static constexpr uint32_t k3 = k65599HashConstant * k2;
+ static constexpr uint32_t k4 = k65599HashConstant * k3;
+ static constexpr uint32_t k5 = k65599HashConstant * k4;
+
+ // Hash a few things at hash length 4
+ EXPECT_EQ(PwTokenizer65599FixedLengthHash("", 4), StringLength(""));
+ EXPECT_EQ(PwTokenizer65599FixedLengthHash("1", 4),
+ StringLength("1") + k2 * '1');
+ EXPECT_EQ(PwTokenizer65599FixedLengthHash("12", 4),
+ StringLength("12") + k2 * '1' + k3 * '2');
+ EXPECT_EQ(PwTokenizer65599FixedLengthHash("123", 4),
+ StringLength("123") + k2 * '1' + k3 * '2' + k4 * '3');
+ EXPECT_EQ(PwTokenizer65599FixedLengthHash("1234", 4),
+ StringLength("1234") + k2 * '1' + k3 * '2' + k4 * '3' + k5 * '4');
+ EXPECT_EQ(PwTokenizer65599FixedLengthHash("12345", 4),
+ StringLength("12345") + k2 * '1' + k3 * '2' + k4 * '3' + k5 * '4');
+ EXPECT_EQ(PwTokenizer65599FixedLengthHash("123456", 4),
+ StringLength("123456") + k2 * '1' + k3 * '2' + k4 * '3' + k5 * '4');
+}
+
+// Use std::string_view so that \0 can appear in strings.
+#define TEST_SUPPORTED_HASHES(string_literal) \
+ static_assert( \
+ PwTokenizer65599FixedLengthHash( \
+ std::string_view(string_literal, sizeof(string_literal) - 1), 80) == \
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH(string_literal), \
+ "80-byte hash mismatch!"); \
+ static_assert( \
+ PwTokenizer65599FixedLengthHash( \
+ std::string_view(string_literal, sizeof(string_literal) - 1), 96) == \
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH(string_literal), \
+ "96-byte hash mismatch!"); \
+ static_assert( \
+ PwTokenizer65599FixedLengthHash( \
+ std::string_view(string_literal, sizeof(string_literal) - 1), \
+ 128) == PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH(string_literal), \
+ "128-byte hash mismatch!")
+
+TEST(HashMacro, Empty) { TEST_SUPPORTED_HASHES(""); }
+
+TEST(HashMacro, NullTerminatorsAreIncludedInHash) {
+ using namespace std::literals; // use an sv literal to include the \0
+
+ TEST_SUPPORTED_HASHES("hello\0there");
+ TEST_SUPPORTED_HASHES("\0");
+ TEST_SUPPORTED_HASHES("\0\0");
+ TEST_SUPPORTED_HASHES("\0\0\0");
+
+ static_assert(
+ PwTokenizer65599FixedLengthHash(""sv, 80) !=
+ PwTokenizer65599FixedLengthHash("\0"sv, 80),
+ "Hashes of \\0 strings should differ since the lengths are included");
+ static_assert(
+ PwTokenizer65599FixedLengthHash("\0"sv, 80) !=
+ PwTokenizer65599FixedLengthHash("\0\0"sv, 80),
+ "Hashes of \\0 strings should differ since the lengths are included");
+ static_assert(
+ PwTokenizer65599FixedLengthHash("\0\0"sv, 80) !=
+ PwTokenizer65599FixedLengthHash("\0\0\0"sv, 80),
+ "Hashes of \\0 strings should differ since the lengths are included");
+}
+
+TEST(HashMacro, OneChar) {
+ TEST_SUPPORTED_HASHES("a");
+ TEST_SUPPORTED_HASHES("c");
+ TEST_SUPPORTED_HASHES("A");
+ TEST_SUPPORTED_HASHES("B");
+ TEST_SUPPORTED_HASHES("C");
+ TEST_SUPPORTED_HASHES("$");
+ TEST_SUPPORTED_HASHES("%");
+ TEST_SUPPORTED_HASHES("^");
+ TEST_SUPPORTED_HASHES("\xa1");
+ TEST_SUPPORTED_HASHES("\xff");
+}
+
+TEST(HashMacro, Phrases) {
+ TEST_SUPPORTED_HASHES("WASD");
+ TEST_SUPPORTED_HASHES("hello, world");
+ TEST_SUPPORTED_HASHES("Let the wookie win.");
+ TEST_SUPPORTED_HASHES("\x01 more test, just for \xffun");
+}
+
+TEST(HashMacro, HashesChange) {
+ static_assert(PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("") !=
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\0"));
+ static_assert(PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("a") !=
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("b"));
+ static_assert(PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("z") !=
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("aa"));
+ static_assert(
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("make sure hashes change") !=
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("MAKE SURE HASHES CHANGE"));
+}
+
+// Define string literals to test boundary conditions.
+#define LITERAL_79 \
+ "0123456789ABCDEF" \
+ "0123456789ABCDEF" \
+ "0123456789ABCDEF" \
+ "0123456789ABCDEF" \
+ "0123456789ABCDE"
+#define LITERAL_80 LITERAL_79 "F"
+
+#define LITERAL_96 LITERAL_80 "0123456789ABCDEF"
+
+#define LITERAL_128 \
+ LITERAL_96 \
+ "0123456789ABCDEF" \
+ "0123456789ABCDEF"
+
+static_assert(sizeof(LITERAL_79) - 1 == 79);
+static_assert(sizeof(LITERAL_80) - 1 == 80);
+static_assert(sizeof(LITERAL_96) - 1 == 96);
+static_assert(sizeof(LITERAL_128) - 1 == 128);
+
+TEST(HashMacro, HashNearFixedHashLength_80) {
+ // 79, 80, 81, 82 bytes
+ TEST_SUPPORTED_HASHES(LITERAL_79);
+ TEST_SUPPORTED_HASHES(LITERAL_80);
+ TEST_SUPPORTED_HASHES(LITERAL_80 "!");
+ TEST_SUPPORTED_HASHES(LITERAL_80 "!!");
+}
+
+TEST(HashMacro, HashNearFixedHashLength_96) {
+ // 95, 96, 97, 98 bytes
+ TEST_SUPPORTED_HASHES(LITERAL_80 "0123456789ABCDE");
+ TEST_SUPPORTED_HASHES(LITERAL_96);
+ TEST_SUPPORTED_HASHES(LITERAL_96 "!");
+ TEST_SUPPORTED_HASHES(LITERAL_96 "XY");
+}
+
+TEST(HashMacro, HashNearFixedHashLength_128) {
+ // 127, 128, 129, 130 bytes
+ TEST_SUPPORTED_HASHES(LITERAL_96
+ "0123456789ABCDEF"
+ "0123456789ABCDE");
+ TEST_SUPPORTED_HASHES(LITERAL_128);
+ TEST_SUPPORTED_HASHES(LITERAL_128 "!");
+ TEST_SUPPORTED_HASHES(LITERAL_128 "AB");
+}
+
+TEST(HashMacro, HashVeryLongStrings) {
+ TEST_SUPPORTED_HASHES(
+ "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
+ "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
+ "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
+ "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"
+ "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0");
+ TEST_SUPPORTED_HASHES(LITERAL_128 LITERAL_128 LITERAL_128);
+}
+
+TEST(HashMacro, ExtraCharactersAreIgnored) {
+ static_assert(PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH(LITERAL_80 "?") ==
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH(LITERAL_80 "."));
+
+ static_assert(PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH(LITERAL_80 "ABCD") ==
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH(LITERAL_80 "zyxw"));
+
+ static_assert(PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH(LITERAL_96 "?") ==
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH(LITERAL_96 "."));
+
+ static_assert(PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH(LITERAL_96 "ABCD") ==
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH(LITERAL_96 "zyxw"));
+
+ static_assert(PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH(LITERAL_128 "?") ==
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH(LITERAL_128 "."));
+
+ static_assert(PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH(LITERAL_128 "ABCD") ==
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH(LITERAL_128 "zyxw"));
+}
+
+} // namespace
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/java/dev/pigweed/tokenizer/Detokenizer.java b/pw_tokenizer/java/dev/pigweed/tokenizer/Detokenizer.java
new file mode 100644
index 0000000..01a21e1
--- /dev/null
+++ b/pw_tokenizer/java/dev/pigweed/tokenizer/Detokenizer.java
@@ -0,0 +1,91 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+/**
+ * This is a Java Native Interface (JNI) wrapper for the Detokenizer class.
+ *
+ * <p>This classes uses the Android Base64 library instead of the standard Java Base64, which is not
+ * yet available on Android. To use this class outside of Android, replace android.util.Base64 with
+ * java.util.Base64.
+ */
+package dev.pigweed.tokenizer;
+
+import android.util.Base64;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/** This class provides the Java interface for the C++ Detokenizer class. */
+public class Detokenizer {
+ // Android's Base64 library doesn't seem to check if the Base64-encoded data is valid. This
+ // regular expression checks that it is. Does not match URL-safe or unpadded Base64.
+ private static final Pattern TOKENIZED_STRING =
+ Pattern.compile("\\$([A-Za-z0-9+/]{4})*([A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?");
+
+ static {
+ System.loadLibrary("detokenizer");
+ }
+
+ // The handle to the C++ detokenizer instance.
+ private final long handle;
+
+ public Detokenizer() {
+ this(new byte[0]);
+ }
+
+ public Detokenizer(byte[] tokenDatabase) {
+ handle = newNativeDetokenizer(tokenDatabase);
+ }
+
+ /**
+ * Detokenizes and replaces all recognized tokenized messages ($ followed by Base64) in the
+ * provided string. Unrecognized tokenized strings are left unchanged.
+ */
+ public String detokenize(String message) {
+ Matcher matcher = TOKENIZED_STRING.matcher(message);
+ StringBuilder result = new StringBuilder();
+ int lastIndex = 0;
+
+ while (matcher.find()) {
+ result.append(message, lastIndex, matcher.start());
+
+ String decoded =
+ detokenizeNative(handle, Base64.decode(matcher.group().substring(1), Base64.DEFAULT));
+ result.append(decoded != null ? decoded : matcher.group());
+
+ lastIndex = matcher.end();
+ }
+
+ result.append(message, lastIndex, message.length());
+ return result.toString();
+ }
+
+ /** Deletes memory allocated in C++ when this class is garbage collected. */
+ @Override
+ protected void finalize() {
+ deleteNativeDetokenizer(handle);
+ }
+
+ /** Creates a new detokenizer using the provided data as the database. */
+ private static native long newNativeDetokenizer(byte[] data);
+
+ /** Deletes the detokenizer object with the provided handle, which MUST be valid. */
+ private static native void deleteNativeDetokenizer(long handle);
+
+ /**
+ * Returns the detokenized version of the provided data. This is non-static so this object has a
+ * reference held while the function is running, which prevents finalize from running before
+ * detokenizeNative finishes.
+ */
+ private native String detokenizeNative(long handle, byte[] data);
+}
diff --git a/pw_tokenizer/java/dev/pigweed/tokenizer/detokenizer.cc b/pw_tokenizer/java/dev/pigweed/tokenizer/detokenizer.cc
new file mode 100644
index 0000000..b2c8f33
--- /dev/null
+++ b/pw_tokenizer/java/dev/pigweed/tokenizer/detokenizer.cc
@@ -0,0 +1,92 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// This file provides a Java Native Interface (JNI) version of the Detokenizer
+// class. This facilitates using the tokenizer library from Java or other JVM
+// languages. A corresponding Java class is provided in Detokenizer.java.
+
+#include <jni.h>
+
+#include <cstring>
+
+#include "pw_preprocessor/concat.h"
+#include "pw_span/span.h"
+#include "pw_tokenizer/detokenize.h"
+#include "pw_tokenizer/token_database.h"
+
+#define DETOKENIZER_METHOD(method) \
+ JNICALL PW_CONCAT(Java_dev_pigweed_tokenizer_, Detokenizer_, method)
+
+namespace pw::tokenizer {
+namespace {
+
+Detokenizer* HandleToPointer(jlong handle) {
+ Detokenizer* detokenizer = nullptr;
+ std::memcpy(&detokenizer, &handle, sizeof(detokenizer));
+ static_assert(sizeof(detokenizer) <= sizeof(handle));
+ return detokenizer;
+}
+
+jlong PointerToHandle(Detokenizer* detokenizer) {
+ jlong handle = 0;
+ std::memcpy(&handle, &detokenizer, sizeof(detokenizer));
+ static_assert(sizeof(handle) >= sizeof(detokenizer));
+ return handle;
+}
+
+} // namespace
+
+extern "C" {
+
+static_assert(sizeof(jbyte) == 1u);
+
+JNIEXPORT jlong DETOKENIZER_METHOD(newNativeDetokenizer)(JNIEnv* env,
+ jclass,
+ jbyteArray array) {
+ jbyte* const data = env->GetByteArrayElements(array, nullptr);
+ const jsize size = env->GetArrayLength(array);
+
+ TokenDatabase tokens = TokenDatabase::Create(pw::span(data, size));
+ const jlong handle =
+ PointerToHandle(new Detokenizer(tokens.ok() ? tokens : TokenDatabase()));
+
+ env->ReleaseByteArrayElements(array, data, 0);
+ return handle;
+}
+
+JNIEXPORT void DETOKENIZER_METHOD(deleteNativeDetokenizer)(JNIEnv*,
+ jclass,
+ jlong handle) {
+ delete HandleToPointer(handle);
+}
+
+JNIEXPORT jstring DETOKENIZER_METHOD(detokenizeNative)(JNIEnv* env,
+ jobject,
+ jlong handle,
+ jbyteArray array) {
+ jbyte* const data = env->GetByteArrayElements(array, nullptr);
+ const jsize size = env->GetArrayLength(array);
+
+ DetokenizedString result = HandleToPointer(handle)->Detokenize(data, size);
+
+ env->ReleaseByteArrayElements(array, data, 0);
+
+ return result.matches().empty()
+ ? nullptr
+ : env->NewStringUTF(result.BestString().c_str());
+}
+
+} // extern "C"
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/public/pw_tokenizer/config.h b/pw_tokenizer/public/pw_tokenizer/config.h
new file mode 100644
index 0000000..d9798d0
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/config.h
@@ -0,0 +1,88 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// Configuration macros for the tokenizer module.
+#pragma once
+
+#include <assert.h>
+
+// TODO(pwbug/17): Configure these options in the config system.
+
+// For a tokenized string that has arguments, the types of the arguments are
+// encoded in either a 4-byte (uint32_t) or a 8-byte (uint64_t) value. The 4 or
+// 6 least-significant bits, respectively, store the number of arguments, while
+// the remaining bits encode the argument types. Argument types are encoded
+// two-bits per argument, in little-endian order. Up to 14 arguments in 4 bytes
+// or 29 arguments in 8 bytes are supported.
+#ifndef PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES
+#define PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES 4
+#endif // PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES
+
+static_assert(PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES == 4 ||
+ PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES == 8,
+ "PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES must be 4 or 8");
+
+// How long of a string to hash. Strings shorter than this length are treated as
+// if they were zero-padded up to the length. Strings that are the same length
+// and share a common prefix longer than this value hash to the same value.
+//
+// Increasing PW_TOKENIZER_CFG_HASH_LENGTH increases the compilation time for C
+// due to the complexity of the hashing macros. C++ macros use a constexpr
+// function instead of a macro, so the compilation time impact is minimal.
+// Projects primarily in C++ should use a large value for
+// PW_TOKENIZER_CFG_HASH_LENGTH (perhaps even
+// std::numeric_limits<size_t>::max()).
+//
+// Only hash lengths for which there is a corresponding macro header
+// (pw_tokenizer/internal/mash_macro_#.h) are supported. Additional macros may
+// be generated with the generate_hash_macro.py function. New macro headers must
+// then be added to pw_tokenizer/internal/hash.h.
+#ifndef PW_TOKENIZER_CFG_HASH_LENGTH
+#define PW_TOKENIZER_CFG_HASH_LENGTH 128
+#endif // PW_TOKENIZER_CFG_HASH_LENGTH
+
+// The size of the stack-allocated argument encoding buffer to use. This only
+// affects tokenization macros that stack-allocate the encoding buffer
+// (PW_TOKENIZE_TO_CALLBACK and PW_TOKENIZE_TO_GLOBAL_HANDLER). This buffer size
+// is only allocated for argument encoding and does not include the 4-byte
+// token.
+//
+// This buffer does not need to be large to accommodate a good number of
+// tokenized string arguments. Integer arguments are usually encoded smaller
+// than their native size (e.g. 1 or 2 bytes for smaller numbers). All floating
+// point types are encoded as four bytes. Null-terminated strings are encoded
+// 1:1 in size.
+#ifndef PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES
+#define PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES 48
+#endif // PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES
+
+// Enables the PW_TOKENIZE_TO_GLOBAL_HANDLER macro. If this option is enabled,
+// the pw_TokenizerHandleEncodedMessage function must be defined by the
+// application.
+#ifndef PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+#define PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER 1
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+// Enables the PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD macro. If this option
+// is enabled, the pw_TokenizerHandleEncodedMessageWithPayload function must be
+// defined by the application.
+#ifndef PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+#define PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD 1
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+
+// Sets the type of the payload argument to use for
+// PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD.
+#ifndef PW_TOKENIZER_CFG_PAYLOAD_TYPE
+#define PW_TOKENIZER_CFG_PAYLOAD_TYPE void*
+#endif // PW_TOKENIZER_CFG_PAYLOAD_TYPE
diff --git a/pw_tokenizer/public/pw_tokenizer/detokenize.h b/pw_tokenizer/public/pw_tokenizer/detokenize.h
new file mode 100644
index 0000000..7450249
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/detokenize.h
@@ -0,0 +1,97 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// This file provides the Detokenizer class, which is used to decode tokenized
+// strings. To use a Detokenizer, load a binary format token database into
+// memory, construct a TokenDatabase, and pass it to a Detokenizer:
+//
+// std::vector data = ReadFile("my_tokenized_strings.db");
+// Detokenizer detok(TokenDatabase::Create(data));
+//
+// DetokenizedString result = detok.Detokenize(my_data);
+// std::cout << result.BestString() << '\n';
+//
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string>
+#include <unordered_map>
+#include <utility>
+#include <vector>
+
+#include "pw_span/span.h"
+#include "pw_tokenizer/internal/decode.h"
+#include "pw_tokenizer/token_database.h"
+
+namespace pw::tokenizer {
+
+using TokenizedStringEntry = std::pair<FormatString, uint32_t /*date removed*/>;
+
+// A string that has been detokenized. This class tracks all possible results if
+// there are token collisions.
+class DetokenizedString {
+ public:
+ DetokenizedString(uint32_t token,
+ const span<const TokenizedStringEntry>& entries,
+ const span<const uint8_t>& arguments);
+
+ DetokenizedString() : has_token_(false) {}
+
+ // True if there was only one valid match and it decoded successfully.
+ bool ok() const { return matches_.size() == 1 && matches_[0].ok(); }
+
+ // Returns the strings that matched the token, with the best matches first.
+ const std::vector<DecodedFormatString>& matches() const { return matches_; }
+
+ // Returns the detokenized string or an empty string if there were no matches.
+ // If there are multiple possible results, the DetokenizedString returns the
+ // first match.
+ std::string BestString() const;
+
+ // Returns the best match, with error messages inserted for arguments that
+ // failed to parse.
+ std::string BestStringWithErrors() const;
+
+ private:
+ uint32_t token_;
+ bool has_token_;
+ std::vector<DecodedFormatString> matches_;
+};
+
+// Decodes and detokenizes strings from a TokenDatabase. This class builds a
+// hash table from the TokenDatabase to give O(1) token lookups.
+class Detokenizer {
+ public:
+ // Constructs a detokenizer from a TokenDatabase. The TokenDatabase is not
+ // referenced by the Detokenizer after construction; its memory can be freed.
+ Detokenizer(const TokenDatabase& database);
+
+ // Decodes and detokenizes the encoded message. Returns a DetokenizedString
+ // that stores all possible detokenized string results.
+ DetokenizedString Detokenize(const span<const uint8_t>& encoded) const;
+
+ DetokenizedString Detokenize(const std::string_view& encoded) const {
+ return Detokenize(encoded.data(), encoded.size());
+ }
+
+ DetokenizedString Detokenize(const void* encoded, size_t size_bytes) const {
+ return Detokenize(span(static_cast<const uint8_t*>(encoded), size_bytes));
+ }
+
+ private:
+ std::unordered_map<uint32_t, std::vector<TokenizedStringEntry>> database_;
+};
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h b/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h
new file mode 100644
index 0000000..4ceea40
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/internal/argument_types.h
@@ -0,0 +1,154 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// This header provides internal macros used by the tokenizer module.
+#pragma once
+
+#include <stdint.h>
+
+#include "pw_preprocessor/macro_arg_count.h"
+#include "pw_tokenizer/config.h"
+
+// The size of the argument types variable determines the number of arguments
+// supported in tokenized strings.
+#if PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES == 4
+
+#include "pw_tokenizer/internal/argument_types_macro_4_byte.h"
+
+// Encoding types in a uint32_t supports 14 arguments with 2 bits per argument.
+#define PW_TOKENIZER_MAX_SUPPORTED_ARGS 14
+#define PW_TOKENIZER_TYPE_COUNT_SIZE_BITS 4u
+#define PW_TOKENIZER_TYPE_COUNT_MASK 0x0Fu
+
+typedef uint32_t pw_TokenizerArgTypes;
+
+#elif PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES == 8
+
+#include "pw_tokenizer/internal/argument_types_macro_8_byte.h"
+
+// Encoding types in a uint64_t supports 29 arguments with 2 bits per argument.
+#define PW_TOKENIZER_MAX_SUPPORTED_ARGS 29
+#define PW_TOKENIZER_TYPE_COUNT_SIZE_BITS 6u
+#define PW_TOKENIZER_TYPE_COUNT_MASK 0x1Fu // only 5 bits will be needed
+
+typedef uint64_t pw_TokenizerArgTypes;
+
+#else
+
+#error "Unsupported value for PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES"
+
+#endif // PW_TOKENIZER_CFG_ARG_TYPES_SIZE_BYTES
+
+// The tokenized string encoding function is a variadic function that works
+// similarly to printf. Instead of a format string, however, the argument types
+// are packed into a pw_TokenizerArgTypes.
+//
+// The four supported argument types are represented by two-bit argument codes.
+// Just four types are required because only printf-compatible arguments are
+// supported, and variadic arguments are further converted to a more limited set
+// of types.
+//
+// char* values cannot be printed as pointers with %p. These arguments are
+// always encoded as strings. To format a char* as an address, cast it to void*
+// or an integer.
+#define PW_TOKENIZER_ARG_TYPE_INT ((pw_TokenizerArgTypes)0)
+#define PW_TOKENIZER_ARG_TYPE_INT64 ((pw_TokenizerArgTypes)1)
+#define PW_TOKENIZER_ARG_TYPE_DOUBLE ((pw_TokenizerArgTypes)2)
+#define PW_TOKENIZER_ARG_TYPE_STRING ((pw_TokenizerArgTypes)3)
+
+// Select the int argument type based on the size of the type. Values smaller
+// than int are promoted to int.
+#define _PW_TOKENIZER_SELECT_INT_TYPE(type) \
+ (sizeof(type) <= sizeof(int) ? PW_TOKENIZER_ARG_TYPE_INT \
+ : PW_TOKENIZER_ARG_TYPE_INT64)
+
+// The _PW_VARARGS_TYPE macro selects the varargs-promoted type at compile time.
+// The macro has to be different for C and C++ because C doesn't support
+// templates and C++ doesn't support _Generic.
+#ifdef __cplusplus
+
+#include <type_traits>
+
+#define _PW_VARARGS_TYPE(arg) ::pw::tokenizer::VarargsType<decltype(arg)>()
+
+namespace pw::tokenizer {
+
+// This function selects the matching type enum for supported argument types.
+template <typename T>
+constexpr pw_TokenizerArgTypes VarargsType() {
+ using ArgType = std::decay_t<T>;
+
+ if constexpr (std::is_floating_point<ArgType>()) {
+ return PW_TOKENIZER_ARG_TYPE_DOUBLE;
+ } else if constexpr (!std::is_null_pointer<ArgType>() &&
+ std::is_convertible<ArgType, const char*>()) {
+ return PW_TOKENIZER_ARG_TYPE_STRING;
+ } else if constexpr (sizeof(ArgType) == sizeof(int64_t)) {
+ return PW_TOKENIZER_ARG_TYPE_INT64;
+ } else {
+ static_assert(sizeof(ArgType) <= sizeof(int));
+ return PW_TOKENIZER_ARG_TYPE_INT;
+ }
+}
+
+} // namespace pw::tokenizer
+
+#else // C version
+
+// This uses a C11 _Generic to select the matching enum value for each supported
+// argument type. _Generic evaluates to the expression matching the type of the
+// provided expression at compile time.
+// clang-format off
+#define _PW_VARARGS_TYPE(arg) \
+ _Generic((arg), \
+ _Bool: PW_TOKENIZER_ARG_TYPE_INT, \
+ char: PW_TOKENIZER_ARG_TYPE_INT, \
+ signed char: PW_TOKENIZER_ARG_TYPE_INT, \
+ unsigned char: PW_TOKENIZER_ARG_TYPE_INT, \
+ signed short: PW_TOKENIZER_ARG_TYPE_INT, \
+ unsigned short: PW_TOKENIZER_ARG_TYPE_INT, \
+ signed int: PW_TOKENIZER_ARG_TYPE_INT, \
+ unsigned int: PW_TOKENIZER_ARG_TYPE_INT, \
+ signed long: _PW_TOKENIZER_SELECT_INT_TYPE(signed long), \
+ unsigned long: _PW_TOKENIZER_SELECT_INT_TYPE(unsigned long), \
+ signed long long: _PW_TOKENIZER_SELECT_INT_TYPE(signed long long), \
+ unsigned long long: _PW_TOKENIZER_SELECT_INT_TYPE(unsigned long long), \
+ float: PW_TOKENIZER_ARG_TYPE_DOUBLE, \
+ double: PW_TOKENIZER_ARG_TYPE_DOUBLE, \
+ long double: PW_TOKENIZER_ARG_TYPE_DOUBLE, \
+ char*: PW_TOKENIZER_ARG_TYPE_STRING, \
+ const char*: PW_TOKENIZER_ARG_TYPE_STRING, \
+ default: _PW_TOKENIZER_SELECT_INT_TYPE(void*))
+// clang-format on
+
+#endif // __cplusplus
+
+// Encodes the types of the provided arguments as a pw_TokenizerArgTypes value.
+// Depending on the size of pw_TokenizerArgTypes, the bottom 4 or 6 bits store
+// the number of arguments and the remaining bits store the types, two bits per
+// type.
+//
+// The arguments are not evaluated; only their types are used to
+// select the set their corresponding PW_TOKENIZER_ARG_TYPEs.
+#define PW_TOKENIZER_ARG_TYPES(...) \
+ _PW_TOKENIZER_TYPES_N(PW_ARG_COUNT(__VA_ARGS__), __VA_ARGS__)
+
+// Selects which _PW_TOKENIZER_TYPES_* macro to use based on the number of
+// arguments this was called with.
+#define _PW_TOKENIZER_TYPES_N(count, ...) \
+ _PW_TOKENIZER_TYPES_EXPAND_N(count, __VA_ARGS__)
+#define _PW_TOKENIZER_TYPES_EXPAND_N(count, ...) \
+ _PW_TOKENIZER_TYPES_##count(__VA_ARGS__)
+
+#define _PW_TOKENIZER_TYPES_0() ((pw_TokenizerArgTypes)0)
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/argument_types_macro_4_byte.h b/pw_tokenizer/public/pw_tokenizer/internal/argument_types_macro_4_byte.h
new file mode 100644
index 0000000..dbb49fd
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/internal/argument_types_macro_4_byte.h
@@ -0,0 +1,59 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// AUTOGENERATED - DO NOT EDIT
+//
+// This file was generated by generate_argument_types_macro.py.
+// To make changes, update the script and run it to generate new files.
+#pragma once
+
+// Macro for encoding tokenizer argument types into an 4-byte value.
+//
+// PW_TOKENIZER_ARG_TYPES could be implemented with recursive macro expansion,
+// but that seems to compile a little slower. Instead, the full macro is
+// generated with Python code. This file is best viewed with line wrapping
+// disabled.
+//
+// These macros depend on macros in pw_tokenizer/internal/argument_types.h.
+// clang-format off
+
+#define _PW_TOKENIZER_TYPES_1(a1) (_PW_VARARGS_TYPE(a1) << 4 | 1)
+
+#define _PW_TOKENIZER_TYPES_2(a1, a2) (_PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 2)
+
+#define _PW_TOKENIZER_TYPES_3(a1, a2, a3) (_PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 3)
+
+#define _PW_TOKENIZER_TYPES_4(a1, a2, a3, a4) (_PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 4)
+
+#define _PW_TOKENIZER_TYPES_5(a1, a2, a3, a4, a5) (_PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 5)
+
+#define _PW_TOKENIZER_TYPES_6(a1, a2, a3, a4, a5, a6) (_PW_VARARGS_TYPE(a6) << 14 | _PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 6)
+
+#define _PW_TOKENIZER_TYPES_7(a1, a2, a3, a4, a5, a6, a7) (_PW_VARARGS_TYPE(a7) << 16 | _PW_VARARGS_TYPE(a6) << 14 | _PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 7)
+
+#define _PW_TOKENIZER_TYPES_8(a1, a2, a3, a4, a5, a6, a7, a8) (_PW_VARARGS_TYPE(a8) << 18 | _PW_VARARGS_TYPE(a7) << 16 | _PW_VARARGS_TYPE(a6) << 14 | _PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 8)
+
+#define _PW_TOKENIZER_TYPES_9(a1, a2, a3, a4, a5, a6, a7, a8, a9) (_PW_VARARGS_TYPE(a9) << 20 | _PW_VARARGS_TYPE(a8) << 18 | _PW_VARARGS_TYPE(a7) << 16 | _PW_VARARGS_TYPE(a6) << 14 | _PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 9)
+
+#define _PW_TOKENIZER_TYPES_10(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) (_PW_VARARGS_TYPE(a10) << 22 | _PW_VARARGS_TYPE(a9) << 20 | _PW_VARARGS_TYPE(a8) << 18 | _PW_VARARGS_TYPE(a7) << 16 | _PW_VARARGS_TYPE(a6) << 14 | _PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 10)
+
+#define _PW_TOKENIZER_TYPES_11(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11) (_PW_VARARGS_TYPE(a11) << 24 | _PW_VARARGS_TYPE(a10) << 22 | _PW_VARARGS_TYPE(a9) << 20 | _PW_VARARGS_TYPE(a8) << 18 | _PW_VARARGS_TYPE(a7) << 16 | _PW_VARARGS_TYPE(a6) << 14 | _PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 11)
+
+#define _PW_TOKENIZER_TYPES_12(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12) (_PW_VARARGS_TYPE(a12) << 26 | _PW_VARARGS_TYPE(a11) << 24 | _PW_VARARGS_TYPE(a10) << 22 | _PW_VARARGS_TYPE(a9) << 20 | _PW_VARARGS_TYPE(a8) << 18 | _PW_VARARGS_TYPE(a7) << 16 | _PW_VARARGS_TYPE(a6) << 14 | _PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 12)
+
+#define _PW_TOKENIZER_TYPES_13(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13) (_PW_VARARGS_TYPE(a13) << 28 | _PW_VARARGS_TYPE(a12) << 26 | _PW_VARARGS_TYPE(a11) << 24 | _PW_VARARGS_TYPE(a10) << 22 | _PW_VARARGS_TYPE(a9) << 20 | _PW_VARARGS_TYPE(a8) << 18 | _PW_VARARGS_TYPE(a7) << 16 | _PW_VARARGS_TYPE(a6) << 14 | _PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 13)
+
+#define _PW_TOKENIZER_TYPES_14(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14) (_PW_VARARGS_TYPE(a14) << 30 | _PW_VARARGS_TYPE(a13) << 28 | _PW_VARARGS_TYPE(a12) << 26 | _PW_VARARGS_TYPE(a11) << 24 | _PW_VARARGS_TYPE(a10) << 22 | _PW_VARARGS_TYPE(a9) << 20 | _PW_VARARGS_TYPE(a8) << 18 | _PW_VARARGS_TYPE(a7) << 16 | _PW_VARARGS_TYPE(a6) << 14 | _PW_VARARGS_TYPE(a5) << 12 | _PW_VARARGS_TYPE(a4) << 10 | _PW_VARARGS_TYPE(a3) << 8 | _PW_VARARGS_TYPE(a2) << 6 | _PW_VARARGS_TYPE(a1) << 4 | 14)
+
+// clang-format on
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/argument_types_macro_8_byte.h b/pw_tokenizer/public/pw_tokenizer/internal/argument_types_macro_8_byte.h
new file mode 100644
index 0000000..1a83256
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/internal/argument_types_macro_8_byte.h
@@ -0,0 +1,89 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// AUTOGENERATED - DO NOT EDIT
+//
+// This file was generated by generate_argument_types_macro.py.
+// To make changes, update the script and run it to generate new files.
+#pragma once
+
+// Macro for encoding tokenizer argument types into an 8-byte value.
+//
+// PW_TOKENIZER_ARG_TYPES could be implemented with recursive macro expansion,
+// but that seems to compile a little slower. Instead, the full macro is
+// generated with Python code. This file is best viewed with line wrapping
+// disabled.
+//
+// These macros depend on macros in pw_tokenizer/internal/argument_types.h.
+// clang-format off
+
+#define _PW_TOKENIZER_TYPES_1(a1) (_PW_VARARGS_TYPE(a1) << 6 | 1)
+
+#define _PW_TOKENIZER_TYPES_2(a1, a2) (_PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 2)
+
+#define _PW_TOKENIZER_TYPES_3(a1, a2, a3) (_PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 3)
+
+#define _PW_TOKENIZER_TYPES_4(a1, a2, a3, a4) (_PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 4)
+
+#define _PW_TOKENIZER_TYPES_5(a1, a2, a3, a4, a5) (_PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 5)
+
+#define _PW_TOKENIZER_TYPES_6(a1, a2, a3, a4, a5, a6) (_PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 6)
+
+#define _PW_TOKENIZER_TYPES_7(a1, a2, a3, a4, a5, a6, a7) (_PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 7)
+
+#define _PW_TOKENIZER_TYPES_8(a1, a2, a3, a4, a5, a6, a7, a8) (_PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 8)
+
+#define _PW_TOKENIZER_TYPES_9(a1, a2, a3, a4, a5, a6, a7, a8, a9) (_PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 9)
+
+#define _PW_TOKENIZER_TYPES_10(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10) (_PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 10)
+
+#define _PW_TOKENIZER_TYPES_11(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11) (_PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 11)
+
+#define _PW_TOKENIZER_TYPES_12(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12) (_PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 12)
+
+#define _PW_TOKENIZER_TYPES_13(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13) (_PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 13)
+
+#define _PW_TOKENIZER_TYPES_14(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14) (_PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 14)
+
+#define _PW_TOKENIZER_TYPES_15(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15) (_PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 15)
+
+#define _PW_TOKENIZER_TYPES_16(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16) (_PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 16)
+
+#define _PW_TOKENIZER_TYPES_17(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17) (_PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 17)
+
+#define _PW_TOKENIZER_TYPES_18(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18) (_PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 18)
+
+#define _PW_TOKENIZER_TYPES_19(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19) (_PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 19)
+
+#define _PW_TOKENIZER_TYPES_20(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) (_PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 20)
+
+#define _PW_TOKENIZER_TYPES_21(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21) (_PW_VARARGS_TYPE(a21) << 46 | _PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 21)
+
+#define _PW_TOKENIZER_TYPES_22(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22) (_PW_VARARGS_TYPE(a22) << 48 | _PW_VARARGS_TYPE(a21) << 46 | _PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 22)
+
+#define _PW_TOKENIZER_TYPES_23(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23) (_PW_VARARGS_TYPE(a23) << 50 | _PW_VARARGS_TYPE(a22) << 48 | _PW_VARARGS_TYPE(a21) << 46 | _PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 23)
+
+#define _PW_TOKENIZER_TYPES_24(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24) (_PW_VARARGS_TYPE(a24) << 52 | _PW_VARARGS_TYPE(a23) << 50 | _PW_VARARGS_TYPE(a22) << 48 | _PW_VARARGS_TYPE(a21) << 46 | _PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 24)
+
+#define _PW_TOKENIZER_TYPES_25(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24, a25) (_PW_VARARGS_TYPE(a25) << 54 | _PW_VARARGS_TYPE(a24) << 52 | _PW_VARARGS_TYPE(a23) << 50 | _PW_VARARGS_TYPE(a22) << 48 | _PW_VARARGS_TYPE(a21) << 46 | _PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 25)
+
+#define _PW_TOKENIZER_TYPES_26(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26) (_PW_VARARGS_TYPE(a26) << 56 | _PW_VARARGS_TYPE(a25) << 54 | _PW_VARARGS_TYPE(a24) << 52 | _PW_VARARGS_TYPE(a23) << 50 | _PW_VARARGS_TYPE(a22) << 48 | _PW_VARARGS_TYPE(a21) << 46 | _PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 26)
+
+#define _PW_TOKENIZER_TYPES_27(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27) (_PW_VARARGS_TYPE(a27) << 58 | _PW_VARARGS_TYPE(a26) << 56 | _PW_VARARGS_TYPE(a25) << 54 | _PW_VARARGS_TYPE(a24) << 52 | _PW_VARARGS_TYPE(a23) << 50 | _PW_VARARGS_TYPE(a22) << 48 | _PW_VARARGS_TYPE(a21) << 46 | _PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 27)
+
+#define _PW_TOKENIZER_TYPES_28(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27, a28) (_PW_VARARGS_TYPE(a28) << 60 | _PW_VARARGS_TYPE(a27) << 58 | _PW_VARARGS_TYPE(a26) << 56 | _PW_VARARGS_TYPE(a25) << 54 | _PW_VARARGS_TYPE(a24) << 52 | _PW_VARARGS_TYPE(a23) << 50 | _PW_VARARGS_TYPE(a22) << 48 | _PW_VARARGS_TYPE(a21) << 46 | _PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 28)
+
+#define _PW_TOKENIZER_TYPES_29(a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20, a21, a22, a23, a24, a25, a26, a27, a28, a29) (_PW_VARARGS_TYPE(a29) << 62 | _PW_VARARGS_TYPE(a28) << 60 | _PW_VARARGS_TYPE(a27) << 58 | _PW_VARARGS_TYPE(a26) << 56 | _PW_VARARGS_TYPE(a25) << 54 | _PW_VARARGS_TYPE(a24) << 52 | _PW_VARARGS_TYPE(a23) << 50 | _PW_VARARGS_TYPE(a22) << 48 | _PW_VARARGS_TYPE(a21) << 46 | _PW_VARARGS_TYPE(a20) << 44 | _PW_VARARGS_TYPE(a19) << 42 | _PW_VARARGS_TYPE(a18) << 40 | _PW_VARARGS_TYPE(a17) << 38 | _PW_VARARGS_TYPE(a16) << 36 | _PW_VARARGS_TYPE(a15) << 34 | _PW_VARARGS_TYPE(a14) << 32 | _PW_VARARGS_TYPE(a13) << 30 | _PW_VARARGS_TYPE(a12) << 28 | _PW_VARARGS_TYPE(a11) << 26 | _PW_VARARGS_TYPE(a10) << 24 | _PW_VARARGS_TYPE(a9) << 22 | _PW_VARARGS_TYPE(a8) << 20 | _PW_VARARGS_TYPE(a7) << 18 | _PW_VARARGS_TYPE(a6) << 16 | _PW_VARARGS_TYPE(a5) << 14 | _PW_VARARGS_TYPE(a4) << 12 | _PW_VARARGS_TYPE(a3) << 10 | _PW_VARARGS_TYPE(a2) << 8 | _PW_VARARGS_TYPE(a1) << 6 | 29)
+
+// clang-format on
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/decode.h b/pw_tokenizer/public/pw_tokenizer/internal/decode.h
new file mode 100644
index 0000000..8cdf690
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/internal/decode.h
@@ -0,0 +1,267 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// decode.h defines classes that implement tokenized string decoding. These
+// classes should not be used directly; instead decode tokenized messages with
+// the Detokenizer class, defined in pw_tokenizer/detokenize.h.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <cstdio>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#include "pw_span/span.h"
+
+// Decoding errors are marked with prefix and suffix so that they stand out from
+// the rest of the decoded strings. These macros are used to build decoding
+// error strings.
+#define PW_TOKENIZER_ARG_DECODING_ERROR_PREFIX "<["
+#define PW_TOKENIZER_ARG_DECODING_ERROR_SUFFIX "]>"
+#define PW_TOKENIZER_ARG_DECODING_ERROR(message) \
+ PW_TOKENIZER_ARG_DECODING_ERROR_PREFIX message \
+ PW_TOKENIZER_ARG_DECODING_ERROR_SUFFIX
+
+namespace pw::tokenizer {
+
+// The status of an argument that was decoded from an encoded tokenized string.
+// This enum should match the values in decode.py's DecodedArg class.
+class ArgStatus {
+ public:
+ // The Code enum tracks issues arise when decoding a tokenized string
+ // argument. Each value is one bit, and an ArgStatus will have multiple bits
+ // set if multiple issues are encountered.
+ enum Code : unsigned {
+ kOk = 0, // Decoding was successful.
+ kMissing = 1, // The argument was not present in the data.
+ kTruncated = 2, // The argument was truncated during encoding.
+ kDecodeError = 4, // An error occurred while decoding the argument.
+ kSkipped = 8, // Argument was skipped due to a previous error.
+ };
+
+ constexpr ArgStatus(Code code = kOk) : status_(code) {}
+
+ // Sets additional status bits.
+ constexpr void Update(ArgStatus status) { status_ |= status.status_; }
+
+ // True if no decoding errors occurred. Truncated is considered OK, since
+ // encoding and decoding still occurs successfully when a string is truncated.
+ constexpr bool ok() const { return status_ == kOk || status_ == kTruncated; }
+
+ // Checks if an error flag is set in the status.
+ constexpr bool HasError(Code code) const { return (status_ & code) != 0u; }
+
+ private:
+ // Since multiple Code bits may be set in an ArgStatus, the status is stored
+ // as an unsigned instead of a Code.
+ unsigned status_;
+};
+
+// An argument decoded from an encoded tokenized message.
+class DecodedArg {
+ public:
+ // Constructs a DecodedArg from a decoded value. The value is formatted into a
+ // string using the provided format string. The number of bytes that were
+ // decoded to get the value are provided in raw_size_bytes.
+ template <typename ArgumentType>
+ static DecodedArg FromValue(const char* format_string,
+ ArgumentType value,
+ size_t raw_size_bytes,
+ ArgStatus arg_status = ArgStatus::kOk);
+
+ // Constructs a DecodedArg that represents a string literal in the format
+ // string (plain text or % character).
+ DecodedArg(const std::string& literal)
+ : value_(literal), raw_data_size_bytes_(0) {}
+
+ // Constructs a DecodedArg that encountered an error during decoding.
+ DecodedArg(ArgStatus error,
+ const std::string_view& spec,
+ size_t raw_size_bytes = 0u,
+ const std::string_view& value = {});
+
+ // This argument's value as a string. If an error occurred while decoding this
+ // argument, value() will be an error message.
+ const std::string& value() const { return value_; }
+
+ // Returns the conversion specification for this argument (e.g. %02x). This is
+ // empty for literals or "%%".
+ const std::string& spec() const { return spec_; }
+
+ // True if this argument decoded successfully.
+ bool ok() const { return status_.ok(); }
+
+ // How many bytes this arg occupied in the encoded arguments.
+ size_t raw_size_bytes() const { return raw_data_size_bytes_; }
+
+ private:
+ DecodedArg(const char* format, size_t raw_size_bytes, ArgStatus status)
+ : spec_(format), raw_data_size_bytes_(raw_size_bytes), status_(status) {}
+
+ std::string value_;
+ std::string spec_;
+ size_t raw_data_size_bytes_;
+ ArgStatus status_;
+};
+
+// Represents a segment of a printf-style format string. Each StringSegment
+// contains either literal text or a format specifier.
+class StringSegment {
+ public:
+ // Parses a format specifier from the text and returns a StringSegment that
+ // represents it. Returns an empty StringSegment if no valid format specifier
+ // was found.
+ static StringSegment ParseFormatSpec(const char* format);
+
+ // Creates a StringSegment that represents a piece of plain text.
+ StringSegment(const std::string_view& text) : StringSegment(text, kLiteral) {}
+
+ // Returns the DecodedArg with this StringSegment decoded according to the
+ // provided arguments.
+ DecodedArg Decode(const span<const uint8_t>& arguments) const;
+
+ // Skips decoding this StringSegment. Literals and %% are expanded as normal.
+ DecodedArg Skip() const;
+
+ bool empty() const { return text_.empty(); }
+
+ const std::string& text() const { return text_; }
+
+ private:
+ enum Type {
+ kLiteral,
+ kPercent, // %% format specifier
+ kString,
+ kSignedInt,
+ kUnsigned32,
+ kUnsigned64,
+ kFloatingPoint,
+ };
+
+ // Varargs-promoted size of args on this machine; only needed for ints or %p.
+ enum ArgSize : bool { k32Bit, k64Bit };
+
+ template <typename T>
+ static constexpr ArgSize VarargSize() {
+ return sizeof(T) == sizeof(int64_t) ? k64Bit : k32Bit;
+ }
+
+ static ArgSize VarargSize(std::array<char, 2> length, char spec);
+
+ StringSegment() : type_(kLiteral) {}
+
+ StringSegment(const std::string_view& text, Type type)
+ : StringSegment(text, type, VarargSize<void*>()) {}
+
+ StringSegment(const std::string_view& text, Type type, ArgSize local_size)
+ : text_(text), type_(type), local_size_(local_size) {}
+
+ DecodedArg DecodeString(const span<const uint8_t>& arguments) const;
+
+ DecodedArg DecodeInteger(const span<const uint8_t>& arguments) const;
+
+ DecodedArg DecodeFloatingPoint(const span<const uint8_t>& arguments) const;
+
+ std::string text_;
+ Type type_;
+ ArgSize local_size_; // Arg size to use for snprintf on this machine.
+};
+
+// The result of decoding a tokenized message with a FormatString. Stores
+// decoded arguments and whether there was any undecoded data. This is returned
+// from a FormatString::Format call.
+class DecodedFormatString {
+ public:
+ DecodedFormatString(std::vector<DecodedArg>&& segments,
+ size_t remaining_bytes)
+ : segments_(std::move(segments)), remaining_bytes_(remaining_bytes) {}
+
+ DecodedFormatString(const DecodedFormatString&) = default;
+ DecodedFormatString(DecodedFormatString&&) = default;
+
+ DecodedFormatString& operator=(const DecodedFormatString&) = default;
+ DecodedFormatString& operator=(DecodedFormatString&&) = default;
+
+ // Returns the decoded format string. If any argument decoding errors
+ // occurred, the % conversion specifiers are included unmodified.
+ std::string value() const;
+
+ // Returns the decoded format string, with error messages for any arguments
+ // that failed to decode.
+ std::string value_with_errors() const;
+
+ bool ok() const { return remaining_bytes() == 0u && decoding_errors() == 0u; }
+
+ // Returns the number of bytes that remained after decoding.
+ size_t remaining_bytes() const { return remaining_bytes_; }
+
+ // Returns the number of arguments in the format string. %% is not included.
+ size_t argument_count() const;
+
+ // Returns the number of arguments that failed to decode.
+ size_t decoding_errors() const;
+
+ private:
+ std::vector<DecodedArg> segments_;
+ size_t remaining_bytes_;
+};
+
+// Represents a printf-style format string. The string is stored as a vector of
+// StringSegments.
+class FormatString {
+ public:
+ // Constructs a FormatString from a null-terminated format string.
+ FormatString(const char* format_string);
+
+ // Formats this format string according to the provided encoded arguments and
+ // returns a string.
+ DecodedFormatString Format(span<const uint8_t> arguments) const;
+
+ DecodedFormatString Format(const std::string_view& arguments) const {
+ return Format(span(reinterpret_cast<const uint8_t*>(arguments.data()),
+ arguments.size()));
+ }
+
+ private:
+ std::vector<StringSegment> segments_;
+};
+
+// Implementation of DecodedArg::FromValue template function.
+template <typename ArgumentType>
+DecodedArg DecodedArg::FromValue(const char* format,
+ ArgumentType value,
+ size_t raw_size_bytes,
+ ArgStatus status) {
+ DecodedArg arg(format, raw_size_bytes, status);
+ const int value_size = std::snprintf(nullptr, 0u, format, value);
+
+ if (value_size < 0) {
+ arg.status_.Update(ArgStatus::kDecodeError);
+ return arg;
+ }
+
+ // Reserve space in the value string for the snprintf call.
+ arg.value_.append(value_size + 1, '\0');
+
+ // Print the value to the string in the reserved space, then pop off the \0.
+ std::snprintf(arg.value_.data(), arg.value_.size(), format, value);
+ arg.value_.pop_back(); // Remove the trailing \0.
+
+ return arg;
+}
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h
new file mode 100644
index 0000000..1c4eb82
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h
@@ -0,0 +1,161 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// AUTOGENERATED - DO NOT EDIT
+//
+// This file was generated by generate_hash_macro.py.
+// To make changes, update the script and run it to regenerate the files.
+#pragma once
+
+#include <stdint.h>
+
+// 128-character version of the tokenizer hash function.
+//
+// The argument must be a string literal. It is concatenated with "" to ensure
+// that this is the case.
+//
+// clang-format off
+
+#define PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH(str) \
+ (uint32_t)(sizeof(str "") - 1 + /* The argument must be a string literal. */ \
+ 0x0001003fu * (uint8_t)str[0] + \
+ 0x007e0f81u * (uint8_t)( 1 < sizeof(str) ? str[ 1] : 0) + \
+ 0x2e86d0bfu * (uint8_t)( 2 < sizeof(str) ? str[ 2] : 0) + \
+ 0x43ec5f01u * (uint8_t)( 3 < sizeof(str) ? str[ 3] : 0) + \
+ 0x162c613fu * (uint8_t)( 4 < sizeof(str) ? str[ 4] : 0) + \
+ 0xd62aee81u * (uint8_t)( 5 < sizeof(str) ? str[ 5] : 0) + \
+ 0xa311b1bfu * (uint8_t)( 6 < sizeof(str) ? str[ 6] : 0) + \
+ 0xd319be01u * (uint8_t)( 7 < sizeof(str) ? str[ 7] : 0) + \
+ 0xb156c23fu * (uint8_t)( 8 < sizeof(str) ? str[ 8] : 0) + \
+ 0x6698cd81u * (uint8_t)( 9 < sizeof(str) ? str[ 9] : 0) + \
+ 0x0d1b92bfu * (uint8_t)( 10 < sizeof(str) ? str[ 10] : 0) + \
+ 0xcc881d01u * (uint8_t)( 11 < sizeof(str) ? str[ 11] : 0) + \
+ 0x7280233fu * (uint8_t)( 12 < sizeof(str) ? str[ 12] : 0) + \
+ 0x50c7ac81u * (uint8_t)( 13 < sizeof(str) ? str[ 13] : 0) + \
+ 0x8da473bfu * (uint8_t)( 14 < sizeof(str) ? str[ 14] : 0) + \
+ 0x4f377c01u * (uint8_t)( 15 < sizeof(str) ? str[ 15] : 0) + \
+ 0xfaa8843fu * (uint8_t)( 16 < sizeof(str) ? str[ 16] : 0) + \
+ 0x33b78b81u * (uint8_t)( 17 < sizeof(str) ? str[ 17] : 0) + \
+ 0x45ac54bfu * (uint8_t)( 18 < sizeof(str) ? str[ 18] : 0) + \
+ 0x7a27db01u * (uint8_t)( 19 < sizeof(str) ? str[ 19] : 0) + \
+ 0xeacfe53fu * (uint8_t)( 20 < sizeof(str) ? str[ 20] : 0) + \
+ 0xae686a81u * (uint8_t)( 21 < sizeof(str) ? str[ 21] : 0) + \
+ 0x563335bfu * (uint8_t)( 22 < sizeof(str) ? str[ 22] : 0) + \
+ 0x6c593a01u * (uint8_t)( 23 < sizeof(str) ? str[ 23] : 0) + \
+ 0xe3f6463fu * (uint8_t)( 24 < sizeof(str) ? str[ 24] : 0) + \
+ 0x5fda4981u * (uint8_t)( 25 < sizeof(str) ? str[ 25] : 0) + \
+ 0xe03916bfu * (uint8_t)( 26 < sizeof(str) ? str[ 26] : 0) + \
+ 0x44cb9901u * (uint8_t)( 27 < sizeof(str) ? str[ 27] : 0) + \
+ 0x871ba73fu * (uint8_t)( 28 < sizeof(str) ? str[ 28] : 0) + \
+ 0xe70d2881u * (uint8_t)( 29 < sizeof(str) ? str[ 29] : 0) + \
+ 0x04bdf7bfu * (uint8_t)( 30 < sizeof(str) ? str[ 30] : 0) + \
+ 0x227ef801u * (uint8_t)( 31 < sizeof(str) ? str[ 31] : 0) + \
+ 0x7540083fu * (uint8_t)( 32 < sizeof(str) ? str[ 32] : 0) + \
+ 0xe3010781u * (uint8_t)( 33 < sizeof(str) ? str[ 33] : 0) + \
+ 0xe4c1d8bfu * (uint8_t)( 34 < sizeof(str) ? str[ 34] : 0) + \
+ 0x24735701u * (uint8_t)( 35 < sizeof(str) ? str[ 35] : 0) + \
+ 0x4f63693fu * (uint8_t)( 36 < sizeof(str) ? str[ 36] : 0) + \
+ 0xf2b5e681u * (uint8_t)( 37 < sizeof(str) ? str[ 37] : 0) + \
+ 0xa144b9bfu * (uint8_t)( 38 < sizeof(str) ? str[ 38] : 0) + \
+ 0x69a8b601u * (uint8_t)( 39 < sizeof(str) ? str[ 39] : 0) + \
+ 0xb685ca3fu * (uint8_t)( 40 < sizeof(str) ? str[ 40] : 0) + \
+ 0xb52bc581u * (uint8_t)( 41 < sizeof(str) ? str[ 41] : 0) + \
+ 0x5b469abfu * (uint8_t)( 42 < sizeof(str) ? str[ 42] : 0) + \
+ 0x111f1501u * (uint8_t)( 43 < sizeof(str) ? str[ 43] : 0) + \
+ 0x4ba72b3fu * (uint8_t)( 44 < sizeof(str) ? str[ 44] : 0) + \
+ 0xc962a481u * (uint8_t)( 45 < sizeof(str) ? str[ 45] : 0) + \
+ 0x33c77bbfu * (uint8_t)( 46 < sizeof(str) ? str[ 46] : 0) + \
+ 0x39d67401u * (uint8_t)( 47 < sizeof(str) ? str[ 47] : 0) + \
+ 0xafc78c3fu * (uint8_t)( 48 < sizeof(str) ? str[ 48] : 0) + \
+ 0xce5a8381u * (uint8_t)( 49 < sizeof(str) ? str[ 49] : 0) + \
+ 0x4bc75cbfu * (uint8_t)( 50 < sizeof(str) ? str[ 50] : 0) + \
+ 0x02ced301u * (uint8_t)( 51 < sizeof(str) ? str[ 51] : 0) + \
+ 0x83e6ed3fu * (uint8_t)( 52 < sizeof(str) ? str[ 52] : 0) + \
+ 0x63136281u * (uint8_t)( 53 < sizeof(str) ? str[ 53] : 0) + \
+ 0xc4463dbfu * (uint8_t)( 54 < sizeof(str) ? str[ 54] : 0) + \
+ 0x8b083201u * (uint8_t)( 55 < sizeof(str) ? str[ 55] : 0) + \
+ 0x69054e3fu * (uint8_t)( 56 < sizeof(str) ? str[ 56] : 0) + \
+ 0x268d4181u * (uint8_t)( 57 < sizeof(str) ? str[ 57] : 0) + \
+ 0xbe441ebfu * (uint8_t)( 58 < sizeof(str) ? str[ 58] : 0) + \
+ 0xf1829101u * (uint8_t)( 59 < sizeof(str) ? str[ 59] : 0) + \
+ 0x0022af3fu * (uint8_t)( 60 < sizeof(str) ? str[ 60] : 0) + \
+ 0xb7c82081u * (uint8_t)( 61 < sizeof(str) ? str[ 61] : 0) + \
+ 0x5ac0ffbfu * (uint8_t)( 62 < sizeof(str) ? str[ 62] : 0) + \
+ 0x553df001u * (uint8_t)( 63 < sizeof(str) ? str[ 63] : 0) + \
+ 0xea3f103fu * (uint8_t)( 64 < sizeof(str) ? str[ 64] : 0) + \
+ 0xb5c3ff81u * (uint8_t)( 65 < sizeof(str) ? str[ 65] : 0) + \
+ 0xbabce0bfu * (uint8_t)( 66 < sizeof(str) ? str[ 66] : 0) + \
+ 0xd53a4f01u * (uint8_t)( 67 < sizeof(str) ? str[ 67] : 0) + \
+ 0xc85a713fu * (uint8_t)( 68 < sizeof(str) ? str[ 68] : 0) + \
+ 0xbf80de81u * (uint8_t)( 69 < sizeof(str) ? str[ 69] : 0) + \
+ 0xff37c1bfu * (uint8_t)( 70 < sizeof(str) ? str[ 70] : 0) + \
+ 0x9077ae01u * (uint8_t)( 71 < sizeof(str) ? str[ 71] : 0) + \
+ 0x3b74d23fu * (uint8_t)( 72 < sizeof(str) ? str[ 72] : 0) + \
+ 0x73febd81u * (uint8_t)( 73 < sizeof(str) ? str[ 73] : 0) + \
+ 0x4931a2bfu * (uint8_t)( 74 < sizeof(str) ? str[ 74] : 0) + \
+ 0xa5f60d01u * (uint8_t)( 75 < sizeof(str) ? str[ 75] : 0) + \
+ 0xe48e333fu * (uint8_t)( 76 < sizeof(str) ? str[ 76] : 0) + \
+ 0x723d9c81u * (uint8_t)( 77 < sizeof(str) ? str[ 77] : 0) + \
+ 0xb9aa83bfu * (uint8_t)( 78 < sizeof(str) ? str[ 78] : 0) + \
+ 0x34b56c01u * (uint8_t)( 79 < sizeof(str) ? str[ 79] : 0) + \
+ 0x64a6943fu * (uint8_t)( 80 < sizeof(str) ? str[ 80] : 0) + \
+ 0x593d7b81u * (uint8_t)( 81 < sizeof(str) ? str[ 81] : 0) + \
+ 0x71a264bfu * (uint8_t)( 82 < sizeof(str) ? str[ 82] : 0) + \
+ 0x5bb5cb01u * (uint8_t)( 83 < sizeof(str) ? str[ 83] : 0) + \
+ 0x5cbdf53fu * (uint8_t)( 84 < sizeof(str) ? str[ 84] : 0) + \
+ 0xc7fe5a81u * (uint8_t)( 85 < sizeof(str) ? str[ 85] : 0) + \
+ 0x921945bfu * (uint8_t)( 86 < sizeof(str) ? str[ 86] : 0) + \
+ 0x39f72a01u * (uint8_t)( 87 < sizeof(str) ? str[ 87] : 0) + \
+ 0x6dd4563fu * (uint8_t)( 88 < sizeof(str) ? str[ 88] : 0) + \
+ 0x5d803981u * (uint8_t)( 89 < sizeof(str) ? str[ 89] : 0) + \
+ 0x3c0f26bfu * (uint8_t)( 90 < sizeof(str) ? str[ 90] : 0) + \
+ 0xee798901u * (uint8_t)( 91 < sizeof(str) ? str[ 91] : 0) + \
+ 0x38e9b73fu * (uint8_t)( 92 < sizeof(str) ? str[ 92] : 0) + \
+ 0xb8c31881u * (uint8_t)( 93 < sizeof(str) ? str[ 93] : 0) + \
+ 0x908407bfu * (uint8_t)( 94 < sizeof(str) ? str[ 94] : 0) + \
+ 0x983ce801u * (uint8_t)( 95 < sizeof(str) ? str[ 95] : 0) + \
+ 0x5efe183fu * (uint8_t)( 96 < sizeof(str) ? str[ 96] : 0) + \
+ 0x78c6f781u * (uint8_t)( 97 < sizeof(str) ? str[ 97] : 0) + \
+ 0xb077e8bfu * (uint8_t)( 98 < sizeof(str) ? str[ 98] : 0) + \
+ 0x56414701u * (uint8_t)( 99 < sizeof(str) ? str[ 99] : 0) + \
+ 0x8111793fu * (uint8_t)(100 < sizeof(str) ? str[100] : 0) + \
+ 0x3c8bd681u * (uint8_t)(101 < sizeof(str) ? str[101] : 0) + \
+ 0xbceac9bfu * (uint8_t)(102 < sizeof(str) ? str[102] : 0) + \
+ 0x4786a601u * (uint8_t)(103 < sizeof(str) ? str[103] : 0) + \
+ 0x4023da3fu * (uint8_t)(104 < sizeof(str) ? str[104] : 0) + \
+ 0xa311b581u * (uint8_t)(105 < sizeof(str) ? str[105] : 0) + \
+ 0xd6dcaabfu * (uint8_t)(106 < sizeof(str) ? str[106] : 0) + \
+ 0x8b0d0501u * (uint8_t)(107 < sizeof(str) ? str[107] : 0) + \
+ 0x3d353b3fu * (uint8_t)(108 < sizeof(str) ? str[108] : 0) + \
+ 0x4b589481u * (uint8_t)(109 < sizeof(str) ? str[109] : 0) + \
+ 0x1f4d8bbfu * (uint8_t)(110 < sizeof(str) ? str[110] : 0) + \
+ 0x3fd46401u * (uint8_t)(111 < sizeof(str) ? str[111] : 0) + \
+ 0x19459c3fu * (uint8_t)(112 < sizeof(str) ? str[112] : 0) + \
+ 0xd4607381u * (uint8_t)(113 < sizeof(str) ? str[113] : 0) + \
+ 0xb73d6cbfu * (uint8_t)(114 < sizeof(str) ? str[114] : 0) + \
+ 0x84dcc301u * (uint8_t)(115 < sizeof(str) ? str[115] : 0) + \
+ 0x7554fd3fu * (uint8_t)(116 < sizeof(str) ? str[116] : 0) + \
+ 0xdd295281u * (uint8_t)(117 < sizeof(str) ? str[117] : 0) + \
+ 0xbfac4dbfu * (uint8_t)(118 < sizeof(str) ? str[118] : 0) + \
+ 0x79262201u * (uint8_t)(119 < sizeof(str) ? str[119] : 0) + \
+ 0xf2635e3fu * (uint8_t)(120 < sizeof(str) ? str[120] : 0) + \
+ 0x04b33181u * (uint8_t)(121 < sizeof(str) ? str[121] : 0) + \
+ 0x599a2ebfu * (uint8_t)(122 < sizeof(str) ? str[122] : 0) + \
+ 0x3bb08101u * (uint8_t)(123 < sizeof(str) ? str[123] : 0) + \
+ 0x3170bf3fu * (uint8_t)(124 < sizeof(str) ? str[124] : 0) + \
+ 0xe9fe1081u * (uint8_t)(125 < sizeof(str) ? str[125] : 0) + \
+ 0xa6070fbfu * (uint8_t)(126 < sizeof(str) ? str[126] : 0) + \
+ 0xeb7be001u * (uint8_t)(127 < sizeof(str) ? str[127] : 0))
+
+// clang-format on
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h
new file mode 100644
index 0000000..857c61f
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h
@@ -0,0 +1,113 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// AUTOGENERATED - DO NOT EDIT
+//
+// This file was generated by generate_hash_macro.py.
+// To make changes, update the script and run it to regenerate the files.
+#pragma once
+
+#include <stdint.h>
+
+// 80-character version of the tokenizer hash function.
+//
+// The argument must be a string literal. It is concatenated with "" to ensure
+// that this is the case.
+//
+// clang-format off
+
+#define PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH(str) \
+ (uint32_t)(sizeof(str "") - 1 + /* The argument must be a string literal. */ \
+ 0x0001003fu * (uint8_t)str[0] + \
+ 0x007e0f81u * (uint8_t)( 1 < sizeof(str) ? str[ 1] : 0) + \
+ 0x2e86d0bfu * (uint8_t)( 2 < sizeof(str) ? str[ 2] : 0) + \
+ 0x43ec5f01u * (uint8_t)( 3 < sizeof(str) ? str[ 3] : 0) + \
+ 0x162c613fu * (uint8_t)( 4 < sizeof(str) ? str[ 4] : 0) + \
+ 0xd62aee81u * (uint8_t)( 5 < sizeof(str) ? str[ 5] : 0) + \
+ 0xa311b1bfu * (uint8_t)( 6 < sizeof(str) ? str[ 6] : 0) + \
+ 0xd319be01u * (uint8_t)( 7 < sizeof(str) ? str[ 7] : 0) + \
+ 0xb156c23fu * (uint8_t)( 8 < sizeof(str) ? str[ 8] : 0) + \
+ 0x6698cd81u * (uint8_t)( 9 < sizeof(str) ? str[ 9] : 0) + \
+ 0x0d1b92bfu * (uint8_t)(10 < sizeof(str) ? str[10] : 0) + \
+ 0xcc881d01u * (uint8_t)(11 < sizeof(str) ? str[11] : 0) + \
+ 0x7280233fu * (uint8_t)(12 < sizeof(str) ? str[12] : 0) + \
+ 0x50c7ac81u * (uint8_t)(13 < sizeof(str) ? str[13] : 0) + \
+ 0x8da473bfu * (uint8_t)(14 < sizeof(str) ? str[14] : 0) + \
+ 0x4f377c01u * (uint8_t)(15 < sizeof(str) ? str[15] : 0) + \
+ 0xfaa8843fu * (uint8_t)(16 < sizeof(str) ? str[16] : 0) + \
+ 0x33b78b81u * (uint8_t)(17 < sizeof(str) ? str[17] : 0) + \
+ 0x45ac54bfu * (uint8_t)(18 < sizeof(str) ? str[18] : 0) + \
+ 0x7a27db01u * (uint8_t)(19 < sizeof(str) ? str[19] : 0) + \
+ 0xeacfe53fu * (uint8_t)(20 < sizeof(str) ? str[20] : 0) + \
+ 0xae686a81u * (uint8_t)(21 < sizeof(str) ? str[21] : 0) + \
+ 0x563335bfu * (uint8_t)(22 < sizeof(str) ? str[22] : 0) + \
+ 0x6c593a01u * (uint8_t)(23 < sizeof(str) ? str[23] : 0) + \
+ 0xe3f6463fu * (uint8_t)(24 < sizeof(str) ? str[24] : 0) + \
+ 0x5fda4981u * (uint8_t)(25 < sizeof(str) ? str[25] : 0) + \
+ 0xe03916bfu * (uint8_t)(26 < sizeof(str) ? str[26] : 0) + \
+ 0x44cb9901u * (uint8_t)(27 < sizeof(str) ? str[27] : 0) + \
+ 0x871ba73fu * (uint8_t)(28 < sizeof(str) ? str[28] : 0) + \
+ 0xe70d2881u * (uint8_t)(29 < sizeof(str) ? str[29] : 0) + \
+ 0x04bdf7bfu * (uint8_t)(30 < sizeof(str) ? str[30] : 0) + \
+ 0x227ef801u * (uint8_t)(31 < sizeof(str) ? str[31] : 0) + \
+ 0x7540083fu * (uint8_t)(32 < sizeof(str) ? str[32] : 0) + \
+ 0xe3010781u * (uint8_t)(33 < sizeof(str) ? str[33] : 0) + \
+ 0xe4c1d8bfu * (uint8_t)(34 < sizeof(str) ? str[34] : 0) + \
+ 0x24735701u * (uint8_t)(35 < sizeof(str) ? str[35] : 0) + \
+ 0x4f63693fu * (uint8_t)(36 < sizeof(str) ? str[36] : 0) + \
+ 0xf2b5e681u * (uint8_t)(37 < sizeof(str) ? str[37] : 0) + \
+ 0xa144b9bfu * (uint8_t)(38 < sizeof(str) ? str[38] : 0) + \
+ 0x69a8b601u * (uint8_t)(39 < sizeof(str) ? str[39] : 0) + \
+ 0xb685ca3fu * (uint8_t)(40 < sizeof(str) ? str[40] : 0) + \
+ 0xb52bc581u * (uint8_t)(41 < sizeof(str) ? str[41] : 0) + \
+ 0x5b469abfu * (uint8_t)(42 < sizeof(str) ? str[42] : 0) + \
+ 0x111f1501u * (uint8_t)(43 < sizeof(str) ? str[43] : 0) + \
+ 0x4ba72b3fu * (uint8_t)(44 < sizeof(str) ? str[44] : 0) + \
+ 0xc962a481u * (uint8_t)(45 < sizeof(str) ? str[45] : 0) + \
+ 0x33c77bbfu * (uint8_t)(46 < sizeof(str) ? str[46] : 0) + \
+ 0x39d67401u * (uint8_t)(47 < sizeof(str) ? str[47] : 0) + \
+ 0xafc78c3fu * (uint8_t)(48 < sizeof(str) ? str[48] : 0) + \
+ 0xce5a8381u * (uint8_t)(49 < sizeof(str) ? str[49] : 0) + \
+ 0x4bc75cbfu * (uint8_t)(50 < sizeof(str) ? str[50] : 0) + \
+ 0x02ced301u * (uint8_t)(51 < sizeof(str) ? str[51] : 0) + \
+ 0x83e6ed3fu * (uint8_t)(52 < sizeof(str) ? str[52] : 0) + \
+ 0x63136281u * (uint8_t)(53 < sizeof(str) ? str[53] : 0) + \
+ 0xc4463dbfu * (uint8_t)(54 < sizeof(str) ? str[54] : 0) + \
+ 0x8b083201u * (uint8_t)(55 < sizeof(str) ? str[55] : 0) + \
+ 0x69054e3fu * (uint8_t)(56 < sizeof(str) ? str[56] : 0) + \
+ 0x268d4181u * (uint8_t)(57 < sizeof(str) ? str[57] : 0) + \
+ 0xbe441ebfu * (uint8_t)(58 < sizeof(str) ? str[58] : 0) + \
+ 0xf1829101u * (uint8_t)(59 < sizeof(str) ? str[59] : 0) + \
+ 0x0022af3fu * (uint8_t)(60 < sizeof(str) ? str[60] : 0) + \
+ 0xb7c82081u * (uint8_t)(61 < sizeof(str) ? str[61] : 0) + \
+ 0x5ac0ffbfu * (uint8_t)(62 < sizeof(str) ? str[62] : 0) + \
+ 0x553df001u * (uint8_t)(63 < sizeof(str) ? str[63] : 0) + \
+ 0xea3f103fu * (uint8_t)(64 < sizeof(str) ? str[64] : 0) + \
+ 0xb5c3ff81u * (uint8_t)(65 < sizeof(str) ? str[65] : 0) + \
+ 0xbabce0bfu * (uint8_t)(66 < sizeof(str) ? str[66] : 0) + \
+ 0xd53a4f01u * (uint8_t)(67 < sizeof(str) ? str[67] : 0) + \
+ 0xc85a713fu * (uint8_t)(68 < sizeof(str) ? str[68] : 0) + \
+ 0xbf80de81u * (uint8_t)(69 < sizeof(str) ? str[69] : 0) + \
+ 0xff37c1bfu * (uint8_t)(70 < sizeof(str) ? str[70] : 0) + \
+ 0x9077ae01u * (uint8_t)(71 < sizeof(str) ? str[71] : 0) + \
+ 0x3b74d23fu * (uint8_t)(72 < sizeof(str) ? str[72] : 0) + \
+ 0x73febd81u * (uint8_t)(73 < sizeof(str) ? str[73] : 0) + \
+ 0x4931a2bfu * (uint8_t)(74 < sizeof(str) ? str[74] : 0) + \
+ 0xa5f60d01u * (uint8_t)(75 < sizeof(str) ? str[75] : 0) + \
+ 0xe48e333fu * (uint8_t)(76 < sizeof(str) ? str[76] : 0) + \
+ 0x723d9c81u * (uint8_t)(77 < sizeof(str) ? str[77] : 0) + \
+ 0xb9aa83bfu * (uint8_t)(78 < sizeof(str) ? str[78] : 0) + \
+ 0x34b56c01u * (uint8_t)(79 < sizeof(str) ? str[79] : 0))
+
+// clang-format on
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h
new file mode 100644
index 0000000..799044c
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h
@@ -0,0 +1,129 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// AUTOGENERATED - DO NOT EDIT
+//
+// This file was generated by generate_hash_macro.py.
+// To make changes, update the script and run it to regenerate the files.
+#pragma once
+
+#include <stdint.h>
+
+// 96-character version of the tokenizer hash function.
+//
+// The argument must be a string literal. It is concatenated with "" to ensure
+// that this is the case.
+//
+// clang-format off
+
+#define PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH(str) \
+ (uint32_t)(sizeof(str "") - 1 + /* The argument must be a string literal. */ \
+ 0x0001003fu * (uint8_t)str[0] + \
+ 0x007e0f81u * (uint8_t)( 1 < sizeof(str) ? str[ 1] : 0) + \
+ 0x2e86d0bfu * (uint8_t)( 2 < sizeof(str) ? str[ 2] : 0) + \
+ 0x43ec5f01u * (uint8_t)( 3 < sizeof(str) ? str[ 3] : 0) + \
+ 0x162c613fu * (uint8_t)( 4 < sizeof(str) ? str[ 4] : 0) + \
+ 0xd62aee81u * (uint8_t)( 5 < sizeof(str) ? str[ 5] : 0) + \
+ 0xa311b1bfu * (uint8_t)( 6 < sizeof(str) ? str[ 6] : 0) + \
+ 0xd319be01u * (uint8_t)( 7 < sizeof(str) ? str[ 7] : 0) + \
+ 0xb156c23fu * (uint8_t)( 8 < sizeof(str) ? str[ 8] : 0) + \
+ 0x6698cd81u * (uint8_t)( 9 < sizeof(str) ? str[ 9] : 0) + \
+ 0x0d1b92bfu * (uint8_t)(10 < sizeof(str) ? str[10] : 0) + \
+ 0xcc881d01u * (uint8_t)(11 < sizeof(str) ? str[11] : 0) + \
+ 0x7280233fu * (uint8_t)(12 < sizeof(str) ? str[12] : 0) + \
+ 0x50c7ac81u * (uint8_t)(13 < sizeof(str) ? str[13] : 0) + \
+ 0x8da473bfu * (uint8_t)(14 < sizeof(str) ? str[14] : 0) + \
+ 0x4f377c01u * (uint8_t)(15 < sizeof(str) ? str[15] : 0) + \
+ 0xfaa8843fu * (uint8_t)(16 < sizeof(str) ? str[16] : 0) + \
+ 0x33b78b81u * (uint8_t)(17 < sizeof(str) ? str[17] : 0) + \
+ 0x45ac54bfu * (uint8_t)(18 < sizeof(str) ? str[18] : 0) + \
+ 0x7a27db01u * (uint8_t)(19 < sizeof(str) ? str[19] : 0) + \
+ 0xeacfe53fu * (uint8_t)(20 < sizeof(str) ? str[20] : 0) + \
+ 0xae686a81u * (uint8_t)(21 < sizeof(str) ? str[21] : 0) + \
+ 0x563335bfu * (uint8_t)(22 < sizeof(str) ? str[22] : 0) + \
+ 0x6c593a01u * (uint8_t)(23 < sizeof(str) ? str[23] : 0) + \
+ 0xe3f6463fu * (uint8_t)(24 < sizeof(str) ? str[24] : 0) + \
+ 0x5fda4981u * (uint8_t)(25 < sizeof(str) ? str[25] : 0) + \
+ 0xe03916bfu * (uint8_t)(26 < sizeof(str) ? str[26] : 0) + \
+ 0x44cb9901u * (uint8_t)(27 < sizeof(str) ? str[27] : 0) + \
+ 0x871ba73fu * (uint8_t)(28 < sizeof(str) ? str[28] : 0) + \
+ 0xe70d2881u * (uint8_t)(29 < sizeof(str) ? str[29] : 0) + \
+ 0x04bdf7bfu * (uint8_t)(30 < sizeof(str) ? str[30] : 0) + \
+ 0x227ef801u * (uint8_t)(31 < sizeof(str) ? str[31] : 0) + \
+ 0x7540083fu * (uint8_t)(32 < sizeof(str) ? str[32] : 0) + \
+ 0xe3010781u * (uint8_t)(33 < sizeof(str) ? str[33] : 0) + \
+ 0xe4c1d8bfu * (uint8_t)(34 < sizeof(str) ? str[34] : 0) + \
+ 0x24735701u * (uint8_t)(35 < sizeof(str) ? str[35] : 0) + \
+ 0x4f63693fu * (uint8_t)(36 < sizeof(str) ? str[36] : 0) + \
+ 0xf2b5e681u * (uint8_t)(37 < sizeof(str) ? str[37] : 0) + \
+ 0xa144b9bfu * (uint8_t)(38 < sizeof(str) ? str[38] : 0) + \
+ 0x69a8b601u * (uint8_t)(39 < sizeof(str) ? str[39] : 0) + \
+ 0xb685ca3fu * (uint8_t)(40 < sizeof(str) ? str[40] : 0) + \
+ 0xb52bc581u * (uint8_t)(41 < sizeof(str) ? str[41] : 0) + \
+ 0x5b469abfu * (uint8_t)(42 < sizeof(str) ? str[42] : 0) + \
+ 0x111f1501u * (uint8_t)(43 < sizeof(str) ? str[43] : 0) + \
+ 0x4ba72b3fu * (uint8_t)(44 < sizeof(str) ? str[44] : 0) + \
+ 0xc962a481u * (uint8_t)(45 < sizeof(str) ? str[45] : 0) + \
+ 0x33c77bbfu * (uint8_t)(46 < sizeof(str) ? str[46] : 0) + \
+ 0x39d67401u * (uint8_t)(47 < sizeof(str) ? str[47] : 0) + \
+ 0xafc78c3fu * (uint8_t)(48 < sizeof(str) ? str[48] : 0) + \
+ 0xce5a8381u * (uint8_t)(49 < sizeof(str) ? str[49] : 0) + \
+ 0x4bc75cbfu * (uint8_t)(50 < sizeof(str) ? str[50] : 0) + \
+ 0x02ced301u * (uint8_t)(51 < sizeof(str) ? str[51] : 0) + \
+ 0x83e6ed3fu * (uint8_t)(52 < sizeof(str) ? str[52] : 0) + \
+ 0x63136281u * (uint8_t)(53 < sizeof(str) ? str[53] : 0) + \
+ 0xc4463dbfu * (uint8_t)(54 < sizeof(str) ? str[54] : 0) + \
+ 0x8b083201u * (uint8_t)(55 < sizeof(str) ? str[55] : 0) + \
+ 0x69054e3fu * (uint8_t)(56 < sizeof(str) ? str[56] : 0) + \
+ 0x268d4181u * (uint8_t)(57 < sizeof(str) ? str[57] : 0) + \
+ 0xbe441ebfu * (uint8_t)(58 < sizeof(str) ? str[58] : 0) + \
+ 0xf1829101u * (uint8_t)(59 < sizeof(str) ? str[59] : 0) + \
+ 0x0022af3fu * (uint8_t)(60 < sizeof(str) ? str[60] : 0) + \
+ 0xb7c82081u * (uint8_t)(61 < sizeof(str) ? str[61] : 0) + \
+ 0x5ac0ffbfu * (uint8_t)(62 < sizeof(str) ? str[62] : 0) + \
+ 0x553df001u * (uint8_t)(63 < sizeof(str) ? str[63] : 0) + \
+ 0xea3f103fu * (uint8_t)(64 < sizeof(str) ? str[64] : 0) + \
+ 0xb5c3ff81u * (uint8_t)(65 < sizeof(str) ? str[65] : 0) + \
+ 0xbabce0bfu * (uint8_t)(66 < sizeof(str) ? str[66] : 0) + \
+ 0xd53a4f01u * (uint8_t)(67 < sizeof(str) ? str[67] : 0) + \
+ 0xc85a713fu * (uint8_t)(68 < sizeof(str) ? str[68] : 0) + \
+ 0xbf80de81u * (uint8_t)(69 < sizeof(str) ? str[69] : 0) + \
+ 0xff37c1bfu * (uint8_t)(70 < sizeof(str) ? str[70] : 0) + \
+ 0x9077ae01u * (uint8_t)(71 < sizeof(str) ? str[71] : 0) + \
+ 0x3b74d23fu * (uint8_t)(72 < sizeof(str) ? str[72] : 0) + \
+ 0x73febd81u * (uint8_t)(73 < sizeof(str) ? str[73] : 0) + \
+ 0x4931a2bfu * (uint8_t)(74 < sizeof(str) ? str[74] : 0) + \
+ 0xa5f60d01u * (uint8_t)(75 < sizeof(str) ? str[75] : 0) + \
+ 0xe48e333fu * (uint8_t)(76 < sizeof(str) ? str[76] : 0) + \
+ 0x723d9c81u * (uint8_t)(77 < sizeof(str) ? str[77] : 0) + \
+ 0xb9aa83bfu * (uint8_t)(78 < sizeof(str) ? str[78] : 0) + \
+ 0x34b56c01u * (uint8_t)(79 < sizeof(str) ? str[79] : 0) + \
+ 0x64a6943fu * (uint8_t)(80 < sizeof(str) ? str[80] : 0) + \
+ 0x593d7b81u * (uint8_t)(81 < sizeof(str) ? str[81] : 0) + \
+ 0x71a264bfu * (uint8_t)(82 < sizeof(str) ? str[82] : 0) + \
+ 0x5bb5cb01u * (uint8_t)(83 < sizeof(str) ? str[83] : 0) + \
+ 0x5cbdf53fu * (uint8_t)(84 < sizeof(str) ? str[84] : 0) + \
+ 0xc7fe5a81u * (uint8_t)(85 < sizeof(str) ? str[85] : 0) + \
+ 0x921945bfu * (uint8_t)(86 < sizeof(str) ? str[86] : 0) + \
+ 0x39f72a01u * (uint8_t)(87 < sizeof(str) ? str[87] : 0) + \
+ 0x6dd4563fu * (uint8_t)(88 < sizeof(str) ? str[88] : 0) + \
+ 0x5d803981u * (uint8_t)(89 < sizeof(str) ? str[89] : 0) + \
+ 0x3c0f26bfu * (uint8_t)(90 < sizeof(str) ? str[90] : 0) + \
+ 0xee798901u * (uint8_t)(91 < sizeof(str) ? str[91] : 0) + \
+ 0x38e9b73fu * (uint8_t)(92 < sizeof(str) ? str[92] : 0) + \
+ 0xb8c31881u * (uint8_t)(93 < sizeof(str) ? str[93] : 0) + \
+ 0x908407bfu * (uint8_t)(94 < sizeof(str) ? str[94] : 0) + \
+ 0x983ce801u * (uint8_t)(95 < sizeof(str) ? str[95] : 0))
+
+// clang-format on
diff --git a/pw_tokenizer/public/pw_tokenizer/internal/tokenize_string.h b/pw_tokenizer/public/pw_tokenizer/internal/tokenize_string.h
new file mode 100644
index 0000000..40e5150
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/internal/tokenize_string.h
@@ -0,0 +1,69 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// Selects the hash macro implementation to use. The implementation selected
+// depends on the language (C or C++) and value of PW_TOKENIZER_CFG_HASH_LENGTH.
+// The options are:
+//
+// - C++ hash constexpr function, which works for any hash length
+// - C 80-character hash macro
+// - C 96-character hash macro
+// - C 128-character hash macro
+//
+// C hash macros for other lengths may be generated using generate_hash_macro.py
+// and added to this file.
+#pragma once
+
+#include <stdint.h>
+
+#if __cplusplus // In C++, use a constexpr function to calculate the hash.
+
+#include "pw_tokenizer/pw_tokenizer_65599_fixed_length_hash.h"
+
+#define PW_TOKENIZER_STRING_TOKEN(format) \
+ pw::tokenizer::PwTokenizer65599FixedLengthHash( \
+ std::string_view((format), sizeof(format "") - 1), \
+ PW_TOKENIZER_CFG_HASH_LENGTH)
+
+#else // In C code, use the hashing macro
+
+#if PW_TOKENIZER_CFG_HASH_LENGTH == 80
+
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h"
+#define PW_TOKENIZER_STRING_TOKEN PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH
+
+#elif PW_TOKENIZER_CFG_HASH_LENGTH == 96
+
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h"
+#define PW_TOKENIZER_STRING_TOKEN PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH
+
+#elif PW_TOKENIZER_CFG_HASH_LENGTH == 128
+
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h"
+#define PW_TOKENIZER_STRING_TOKEN PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH
+
+#else // unsupported hash length
+
+// Only hash lengths for which there is a corresponding macro header
+// (pw_tokenizer/internal/mash_macro_#.h) are supported. Additional macros may
+// be generated with the generate_hash_macro.py function. New macro headers must
+// be added to this file.
+#error "Unsupported value for PW_TOKENIZER_CFG_HASH_LENGTH"
+
+#endif // PW_TOKENIZER_CFG_HASH_LENGTH
+
+#endif // __cplusplus
+
+// The type of the token used in place of a format string.
+typedef uint32_t pw_TokenizerStringToken;
diff --git a/pw_tokenizer/public/pw_tokenizer/pw_tokenizer_65599_fixed_length_hash.h b/pw_tokenizer/public/pw_tokenizer/pw_tokenizer_65599_fixed_length_hash.h
new file mode 100644
index 0000000..bd4a140
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/pw_tokenizer_65599_fixed_length_hash.h
@@ -0,0 +1,59 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string_view>
+
+namespace pw::tokenizer {
+
+// The constant to use when generating the hash. Changing this changes the value
+// of all hashes, so do not change it randomly.
+inline constexpr uint32_t k65599HashConstant = 65599u;
+
+// Calculates the hash of a string. This function calculates hashes at either
+// runtime or compile time in C++ code.
+//
+// This function only hashes up to a fixed length. Characters beyond that length
+// are ignored. Hashing to a fixed length makes it possible to compute this hash
+// in a preprocessor macro. To eliminate some collisions, the length of the
+// string is hashed as if it were the first character.
+//
+// This hash is calculated with the following equation, where s is the string
+// and k is the maximum hash length:
+//
+// H(s, k) = len(s) + 65599 * s[0] + 65599^2 * s[1] + ... + 65599^k * s[k-1]
+//
+// The hash algorithm is a modified version of the x65599 hash used by the SDBM
+// open source project. This hash has the following differences from x65599:
+// - Characters are only hashed up to a fixed maximum string length.
+// - Characters are hashed in reverse order.
+// - The string length is hashed as the first character in the string.
+constexpr uint32_t PwTokenizer65599FixedLengthHash(std::string_view string,
+ size_t hash_length) {
+ // The length is hashed as if it were the first character.
+ uint32_t hash = string.size();
+ uint32_t coefficient = k65599HashConstant;
+
+ // Hash all of the characters in the string as unsigned ints.
+ for (uint8_t ch : string.substr(0, hash_length)) {
+ hash += coefficient * ch;
+ coefficient *= k65599HashConstant;
+ }
+
+ return hash;
+}
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/public/pw_tokenizer/token_database.h b/pw_tokenizer/public/pw_tokenizer/token_database.h
new file mode 100644
index 0000000..ae1ebaf
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/token_database.h
@@ -0,0 +1,308 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <array>
+#include <cstddef>
+#include <cstdint>
+#include <iterator>
+
+namespace pw::tokenizer {
+
+// Reads entries from a binary token string database. This class does not copy
+// or modify the contents of the database.
+//
+// A binary token database is comprised of a 16-byte header followed by an array
+// of 8-byte entries and a table of null-terminated strings. The header
+// specifies the number of entries. Each entry contains information about a
+// tokenized string: the token and removal date, if any. All fields are
+// little-endian.
+//
+// Header
+// ======
+// Offset Size Field
+// -----------------------------------
+// 0 6 Magic number (TOKENS)
+// 6 2 Version (00 00)
+// 8 4 Entry count
+// 12 4 Reserved
+//
+// Entry
+// =====
+// Offset Size Field
+// -----------------------------------
+// 0 4 Token
+// 4 4 Removal date (d m yy)
+//
+// Entries are sorted by token. A string table with a null-terminated string for
+// each entry in order follows the entries.
+//
+// Entries are accessed by iterating over the database. A O(n) Find function is
+// also provided. In typical use, a TokenDatabase is preprocessed by a
+// Detokenizer into a std::unordered_map.
+class TokenDatabase {
+ public:
+ // Internal struct that describes how the underlying binary token database
+ // stores entries. RawEntries generally should not be used directly. Instead,
+ // use an Entry, which contains a pointer to the entry's string.
+ struct RawEntry {
+ uint32_t token;
+ uint32_t date_removed;
+ };
+
+ static_assert(sizeof(RawEntry) == 8u);
+
+ // An entry in the token database. This struct adds the string to a RawEntry.
+ struct Entry {
+ // The token calculated for this string.
+ uint32_t token;
+
+ // The date the token and string was removed from the database, or
+ // 0xFFFFFFFF if it was never removed. Dates are encoded such that natural
+ // integer sorting sorts from oldest to newest dates. The day is stored an
+ // an 8-bit day, 8-bit month, and 16-bit year, packed into a little-endian
+ // uint32_t.
+ uint32_t date_removed;
+
+ // The null-terminated string represented by this token.
+ const char* string;
+ };
+
+ // Iterator for TokenDatabase values. Note that this is not a normal STL-style
+ // iterator, since * returns a value instead of a reference.
+ class Iterator {
+ public:
+ constexpr Iterator(const RawEntry* raw_entry, const char* string)
+ : raw_(raw_entry), string_(string) {}
+
+ // Constructs a TokenDatabase::Entry for the entry this iterator refers to.
+ constexpr Entry entry() const {
+ return {raw_->token, raw_->date_removed, string_};
+ }
+
+ constexpr Iterator& operator++() {
+ raw_ += 1;
+ // Move string_ to the character beyond the next null terminator.
+ while (*string_++ != '\0') {
+ }
+ return *this;
+ }
+ constexpr Iterator operator++(int) { return operator++(); }
+ constexpr bool operator==(const Iterator& rhs) const {
+ return raw_ == rhs.raw_;
+ }
+ constexpr bool operator!=(const Iterator& rhs) const {
+ return raw_ != rhs.raw_;
+ }
+
+ // Derefencing a TokenDatabase::Iterator returns an Entry, not an Entry&.
+ constexpr Entry operator*() const { return entry(); }
+
+ // Point directly into the underlying RawEntry. Strings are not accessible
+ // via this operator.
+ constexpr const RawEntry* operator->() const { return raw_; }
+
+ constexpr ptrdiff_t operator-(const Iterator& rhs) const {
+ return raw_ - rhs.raw_;
+ }
+
+ private:
+ const RawEntry* raw_;
+ const char* string_;
+ };
+
+ // A list of token entries returned from a Find operation. This object can be
+ // iterated over or indexed as an array.
+ class Entries {
+ public:
+ constexpr Entries(const Iterator& begin, const Iterator& end)
+ : begin_(begin), end_(end) {}
+
+ // The number of entries in this list.
+ constexpr size_t size() const { return end_ - begin_; }
+
+ // True of the list is empty.
+ constexpr bool empty() const { return begin_ == end_; }
+
+ // Accesses the specified entry in this set. Returns an Entry object, which
+ // is constructed from the underlying raw entry. The index must be less than
+ // size(). This operation is O(n) in size().
+ Entry operator[](size_t index) const;
+
+ constexpr const Iterator& begin() const { return begin_; }
+ constexpr const Iterator& end() const { return end_; }
+
+ private:
+ Iterator begin_;
+ Iterator end_;
+ };
+
+ // Returns true if the provided data is a valid token database. This checks
+ // the magic number ("TOKENS"), version (which must be 0), and that there is
+ // is one string for each entry in the database. A database with extra strings
+ // or other trailing data is considered valid.
+ template <typename ByteArray>
+ static constexpr bool IsValid(const ByteArray& bytes) {
+ return HasValidHeader(bytes) && EachEntryHasAString(bytes);
+ }
+
+ // Creates a TokenDatabase and checks if the provided data is valid at compile
+ // time. Accepts references to constexpr containers (array, span, string_view,
+ // etc.) with static storage duration. For example:
+ //
+ // constexpr char kMyData[] = ...;
+ // constexpr TokenDatabase db = TokenDatabase::Create<kMyData>();
+ //
+ template <const auto& kDatabaseBytes>
+ static constexpr TokenDatabase Create() {
+ static_assert(
+ HasValidHeader<decltype(kDatabaseBytes)>(kDatabaseBytes),
+ "Databases must start with a 16-byte header that begins with TOKENS.");
+
+ static_assert(EachEntryHasAString<decltype(kDatabaseBytes)>(kDatabaseBytes),
+ "The database must have at least one string for each entry.");
+
+ return TokenDatabase(std::data(kDatabaseBytes));
+ }
+
+ // Creates a TokenDatabase from the provided byte array. The array may be a
+ // span, array, or other container type. If the data is not valid, returns a
+ // default-constructed database for which ok() is false.
+ //
+ // Prefer the Create overload that takes the data as a template parameter
+ // whenever possible, since that function checks the integrity of the data at
+ // compile time.
+ template <typename ByteArray>
+ static constexpr TokenDatabase Create(const ByteArray& database_bytes) {
+ return IsValid<ByteArray>(database_bytes)
+ ? TokenDatabase(std::data(database_bytes))
+ : TokenDatabase(); // Invalid database.
+ }
+ // Creates a database with no data. ok() returns false.
+ constexpr TokenDatabase() : begin_{.data = nullptr}, end_{.data = nullptr} {}
+
+ // Returns all entries associated with this token. This is a O(n) operation.
+ Entries Find(uint32_t token) const;
+
+ // Returns the total number of entries (unique token-string pairs).
+ constexpr size_t size() const {
+ return (end_.data - begin_.data) / sizeof(RawEntry);
+ }
+
+ // True if this database was constructed with valid data.
+ constexpr bool ok() const { return begin_.data != nullptr; }
+
+ Iterator begin() const { return Iterator(begin_.entry, end_.data); }
+ Iterator end() const { return Iterator(end_.entry, nullptr); }
+
+ private:
+ struct Header {
+ std::array<char, 6> magic;
+ uint16_t version;
+ uint32_t entry_count;
+ uint32_t reserved;
+ };
+
+ static_assert(sizeof(Header) == 2 * sizeof(RawEntry));
+
+ template <typename ByteArray>
+ static constexpr bool HasValidHeader(const ByteArray& bytes) {
+ static_assert(sizeof(*std::data(bytes)) == 1u);
+
+ if (std::size(bytes) < sizeof(Header)) {
+ return false;
+ }
+
+ // Check the magic number and version.
+ for (size_t i = 0; i < kMagicAndVersion.size(); ++i) {
+ if (bytes[i] != kMagicAndVersion[i]) {
+ return false;
+ }
+ }
+
+ return true;
+ }
+
+ template <typename ByteArray>
+ static constexpr bool EachEntryHasAString(const ByteArray& bytes) {
+ const size_t entries = ReadEntryCount(std::data(bytes));
+
+ // Check that the data is large enough to have a string table.
+ if (std::size(bytes) < StringTable(entries)) {
+ return false;
+ }
+
+ // Count the strings in the string table.
+ size_t string_count = 0;
+ for (auto i = std::begin(bytes) + StringTable(entries); i < std::end(bytes);
+ ++i) {
+ string_count += (*i == '\0') ? 1 : 0;
+ }
+
+ // Check that there is at least one string for each entry.
+ return string_count >= entries;
+ }
+
+ // Reads the number of entries from a database header. Cast to the bytes to
+ // uint8_t to avoid sign extension if T is signed.
+ template <typename T>
+ static constexpr uint32_t ReadEntryCount(const T* header_bytes) {
+ const T* bytes = header_bytes + offsetof(Header, entry_count);
+ return static_cast<uint8_t>(bytes[0]) |
+ static_cast<uint8_t>(bytes[1]) << 8 |
+ static_cast<uint8_t>(bytes[2]) << 16 |
+ static_cast<uint8_t>(bytes[3]) << 24;
+ }
+
+ // Calculates the offset of the string table.
+ static constexpr size_t StringTable(size_t entries) {
+ return sizeof(Header) + entries * sizeof(RawEntry);
+ }
+
+ // The magic number that starts the table is "TOKENS". The version is encoded
+ // next as two bytes.
+ static constexpr std::array<char, 8> kMagicAndVersion = {
+ 'T', 'O', 'K', 'E', 'N', 'S', '\0', '\0'};
+
+ template <typename Byte>
+ constexpr TokenDatabase(const Byte bytes[])
+ : TokenDatabase(bytes + sizeof(Header),
+ bytes + StringTable(ReadEntryCount(bytes))) {
+ static_assert(sizeof(Byte) == 1u);
+ }
+
+ // It is illegal to reinterpret_cast in constexpr functions, but acceptable to
+ // use unions. Instead of using a reinterpret_cast to change the byte pointer
+ // to a RawEntry pointer, have a separate overload for each byte pointer type
+ // and store them in a union.
+ constexpr TokenDatabase(const char* begin, const char* end)
+ : begin_{.data = begin}, end_{.data = end} {}
+
+ constexpr TokenDatabase(const unsigned char* begin, const unsigned char* end)
+ : begin_{.unsigned_data = begin}, end_{.unsigned_data = end} {}
+
+ constexpr TokenDatabase(const signed char* begin, const signed char* end)
+ : begin_{.signed_data = begin}, end_{.signed_data = end} {}
+
+ // Store the beginning and end pointers as a union to avoid breaking constexpr
+ // rules for reinterpret_cast.
+ union {
+ const RawEntry* entry;
+ const char* data;
+ const unsigned char* unsigned_data;
+ const signed char* signed_data;
+ } begin_, end_;
+};
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/public/pw_tokenizer/tokenize.h b/pw_tokenizer/public/pw_tokenizer/tokenize.h
new file mode 100644
index 0000000..e793d16
--- /dev/null
+++ b/pw_tokenizer/public/pw_tokenizer/tokenize.h
@@ -0,0 +1,350 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <assert.h>
+#include <stddef.h>
+#include <stdint.h>
+
+#include "pw_preprocessor/compiler.h"
+#include "pw_preprocessor/concat.h"
+#include "pw_preprocessor/macro_arg_count.h"
+#include "pw_preprocessor/util.h"
+#include "pw_tokenizer/internal/argument_types.h"
+#include "pw_tokenizer/internal/tokenize_string.h"
+
+// Tokenizes a string literal and converts it to a pw_TokenizerStringToken. This
+// expression can be assigned to a local or global variable, but cannot be used
+// in another expression. For example:
+//
+// constexpr uint32_t global = PW_TOKENIZE_STRING("Wow!"); // This works.
+//
+// void SomeFunction() {
+// constexpr uint32_t token = PW_TOKENIZE_STRING("Cool!"); // This works.
+//
+// DoSomethingElse(PW_TOKENIZE_STRING("Lame!")); // This does NOT work.
+// }
+//
+#define PW_TOKENIZE_STRING(string_literal) \
+ _PW_TOKENIZE_LITERAL_UNIQUE(__COUNTER__, string_literal)
+
+// Encodes a tokenized string and arguments to the provided buffer. The size of
+// the buffer is passed via a pointer to a size_t. After encoding is complete,
+// the size_t is set to the number of bytes written to the buffer.
+//
+// The macro's arguments are equivalent to the following function signature:
+//
+// TokenizeToBuffer(void* buffer,
+// size_t* buffer_size_pointer,
+// const char* format,
+// ...); /* printf-style arguments */
+//
+// For example, the following encodes a tokenized string with a temperature to a
+// buffer. The buffer is passed to a function to send the message over a UART.
+//
+// uint8_t buffer[32];
+// size_t size_bytes = sizeof(buffer);
+// PW_TOKENIZE_TO_BUFFER(
+// buffer, &size_bytes, "Temperature (C): %0.2f", temperature_c);
+// MyProject_EnqueueMessageForUart(buffer, size);
+//
+#define PW_TOKENIZE_TO_BUFFER(buffer, buffer_size_pointer, format, ...) \
+ do { \
+ _PW_TOKENIZE_STRING(format, __VA_ARGS__); \
+ pw_TokenizeToBuffer(buffer, \
+ buffer_size_pointer, \
+ _pw_tokenizer_token, \
+ PW_TOKENIZER_ARG_TYPES(__VA_ARGS__) \
+ PW_COMMA_ARGS(__VA_ARGS__)); \
+ } while (0)
+
+// Encodes a tokenized string and arguments to a buffer on the stack. The
+// provided callback is called with the encoded data. The size of the
+// stack-allocated argument encoding buffer is set with the
+// PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES option.
+//
+// The macro's arguments are equivalent to the following function signature:
+//
+// TokenizeToCallback(void (*callback)(const uint8_t* data, size_t size),
+// const char* format,
+// ...); /* printf-style arguments */
+//
+// For example, the following encodes a tokenized string with a sensor name and
+// floating point data. The encoded message is passed directly to the
+// MyProject_EnqueueMessageForUart function, which the caller provides as a
+// callback.
+//
+// void MyProject_EnqueueMessageForUart(const uint8_t* buffer,
+// size_t size_bytes) {
+// uart_queue_write(uart_instance, buffer, size_bytes);
+// }
+//
+// void LogSensorValue(const char* sensor_name, float value) {
+// PW_TOKENIZE_TO_CALLBACK(MyProject_EnqueueMessageForUart,
+// "%s: %f",
+// sensor_name,
+// value);
+// }
+//
+#define PW_TOKENIZE_TO_CALLBACK(callback, format, ...) \
+ do { \
+ _PW_TOKENIZE_STRING(format, __VA_ARGS__); \
+ pw_TokenizeToCallback(callback, \
+ _pw_tokenizer_token, \
+ PW_TOKENIZER_ARG_TYPES(__VA_ARGS__) \
+ PW_COMMA_ARGS(__VA_ARGS__)); \
+ } while (0)
+
+// Encodes a tokenized string and arguments to a buffer on the stack. The buffer
+// is passed to the user-defined pw_TokenizerHandleEncodedMessage function. The
+// size of the stack-allocated argument encoding buffer is set with the
+// PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES option.
+//
+// The option PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER must be enabled
+// in order to use this macro. When that option is enabled, the
+// pw_TokenizerHandleEncodedMessage function *MUST* be defined by the user of
+// pw_tokenizer.
+//
+// The macro's arguments are equivalent to the following function signature:
+//
+// TokenizeToGlobalHandler(const char* format,
+// ...); /* printf-style arguments */
+//
+// For example, the following encodes a tokenized string with a value returned
+// from a function call. The encoded message is passed to the caller-defined
+// pw_TokenizerHandleEncodedMessage function.
+//
+// void OutputLastReadSize() {
+// PW_TOKENIZE_TO_GLOBAL_HANDLER("Read %u bytes", ReadSizeBytes());
+// }
+//
+// void pw_TokenizerHandleEncodedMessage(const uint8_t encoded_message[],
+// size_t size_bytes) {
+// MyProject_EnqueueMessageForUart(buffer, size_bytes);
+// }
+//
+#if PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+#define PW_TOKENIZE_TO_GLOBAL_HANDLER(format, ...) \
+ do { \
+ _PW_TOKENIZE_STRING(format, __VA_ARGS__); \
+ pw_TokenizeToGlobalHandler(_pw_tokenizer_token, \
+ PW_TOKENIZER_ARG_TYPES(__VA_ARGS__) \
+ PW_COMMA_ARGS(__VA_ARGS__)); \
+ } while (0)
+
+// If PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER is set, this function
+// must be defined by the user of pw_tokenizer. This function is called with the
+// encoded message by pw_TokenizeToGlobalHandler.
+PW_EXTERN_C void pw_TokenizerHandleEncodedMessage(
+ const uint8_t encoded_message[], size_t size_bytes);
+
+// This function encodes the tokenized strings. Do not call it directly;
+// instead, use the PW_TOKENIZE_TO_GLOBAL_HANDLER macro.
+PW_EXTERN_C void pw_TokenizeToGlobalHandler(pw_TokenizerStringToken token,
+ pw_TokenizerArgTypes types,
+ ...);
+
+#else // PW_TOKENIZE_TO_GLOBAL_HANDLER is not available
+
+#define PW_TOKENIZE_TO_GLOBAL_HANDLER(...) \
+ static_assert(0, \
+ "PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER must be " \
+ "set to 1 to use PW_TOKENIZE_TO_GLOBAL_HANDLER")
+
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+// Like PW_TOKENIZE_TO_GLOBAL_HANDLER, encodes a tokenized string and arguments
+// to a buffer on the stack. The macro adds a payload argument, which is passed
+// through to the global handler function
+// pw_TokenizerHandleEncodedMessageWithPayload, which must be defined by the
+// user of pw_tokenizer. The payload type is specified by the
+// PW_TOKENIZER_CFG_PAYLOAD_TYPE option and defaults to void*.
+//
+// For example, the following tokenizes a log string and passes the log level as
+// the payload.
+/*
+ #define LOG_ERROR(...) \
+ PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD(kLogLevelError, __VA_ARGS__)
+
+ void pw_TokenizerHandleEncodedMessageWithPayload(
+ pw_TokenizerPayload log_level,
+ const uint8_t encoded_message[],
+ size_t size_bytes) {
+ if (log_level >= kLogLevelWarning) {
+ MyProject_EnqueueMessageForUart(buffer, size_bytes);
+ }
+ }
+ */
+#if PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+#define PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD(payload, format, ...) \
+ do { \
+ _PW_TOKENIZE_STRING(format, __VA_ARGS__); \
+ pw_TokenizeToGlobalHandlerWithPayload(payload, \
+ _pw_tokenizer_token, \
+ PW_TOKENIZER_ARG_TYPES(__VA_ARGS__) \
+ PW_COMMA_ARGS(__VA_ARGS__)); \
+ } while (0)
+
+typedef PW_TOKENIZER_CFG_PAYLOAD_TYPE pw_TokenizerPayload;
+
+// If PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD is set,
+// this function must be defined by the user of pw_tokenizer. This function is
+// called with the encoded message by pw_TokenizeToGlobalHandler and a
+// caller-provided payload argument.
+PW_EXTERN_C void pw_TokenizerHandleEncodedMessageWithPayload(
+ pw_TokenizerPayload payload,
+ const uint8_t encoded_message[],
+ size_t size_bytes);
+
+// This function encodes the tokenized strings. Do not call it directly;
+// instead, use the PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD macro.
+PW_EXTERN_C void pw_TokenizeToGlobalHandlerWithPayload(
+ pw_TokenizerPayload payload,
+ pw_TokenizerStringToken token,
+ pw_TokenizerArgTypes types,
+ ...);
+
+#else // PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD is not available
+
+#define PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD(...) \
+ static_assert(0, \
+ "PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_" \
+ "PAYLOAD must be set to 1 to use " \
+ "PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD")
+
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+
+PW_EXTERN_C_START
+
+// These functions encode the tokenized strings. These should not be called
+// directly. Instead, use the corresponding PW_TOKENIZE_TO_* macros above.
+void pw_TokenizeToBuffer(void* buffer,
+ size_t* buffer_size_bytes, // input and output arg
+ pw_TokenizerStringToken token,
+ pw_TokenizerArgTypes types,
+ ...);
+
+void pw_TokenizeToCallback(void (*callback)(const uint8_t* encoded_message,
+ size_t size_bytes),
+ pw_TokenizerStringToken token,
+ pw_TokenizerArgTypes types,
+ ...);
+
+// This empty function allows the compiler to check the format string.
+inline void pw_TokenizerCheckFormatString(const char* format, ...)
+ PW_PRINTF_FORMAT(1, 2);
+
+inline void pw_TokenizerCheckFormatString(const char* format, ...) {
+ PW_UNUSED(format);
+}
+
+PW_EXTERN_C_END
+
+// These macros implement string tokenization. They should not be used directly;
+// use one of the PW_TOKENIZE_* macros above instead.
+#define _PW_TOKENIZE_LITERAL_UNIQUE(id, string_literal) \
+ /* assign to a variable */ PW_TOKENIZER_STRING_TOKEN(string_literal); \
+ \
+ /* Declare without nested scope so this works in or out of a function. */ \
+ static _PW_TOKENIZER_CONST char PW_CONCAT( \
+ _pw_tokenizer_string_literal_DO_NOT_USE_THIS_VARIABLE_, \
+ id)[] _PW_TOKENIZER_SECTION(id) = string_literal
+
+// This macro uses __COUNTER__ to generate an identifier to use in the main
+// tokenization macro below. The identifier is unique within a compilation unit.
+#define _PW_TOKENIZE_STRING(format, ...) \
+ _PW_TOKENIZE_STRING_UNIQUE(__COUNTER__, format, __VA_ARGS__)
+
+// This macro takes a printf-style format string and corresponding arguments. It
+// checks that the arguments are correct, stores the format string in a special
+// section, and calculates the string's token at compile time.
+// clang-format off
+#define _PW_TOKENIZE_STRING_UNIQUE(id, format, ...) \
+ if (0) { /* Do not execute to prevent double evaluation of the arguments. */ \
+ pw_TokenizerCheckFormatString(format PW_COMMA_ARGS(__VA_ARGS__)); \
+ } \
+ \
+ /* Check that the macro is invoked with a supported number of arguments. */ \
+ static_assert( \
+ PW_ARG_COUNT(__VA_ARGS__) <= PW_TOKENIZER_MAX_SUPPORTED_ARGS, \
+ "Tokenized strings cannot have more than " \
+ PW_STRINGIFY(PW_TOKENIZER_MAX_SUPPORTED_ARGS) " arguments; " \
+ PW_STRINGIFY(PW_ARG_COUNT(__VA_ARGS__)) " arguments were used for " \
+ #format " (" #__VA_ARGS__ ")"); \
+ \
+ /* Declare the format string as an array in the special tokenized string */ \
+ /* section, which should be excluded from the final binary. Use unique */ \
+ /* names for the section and variable to avoid compiler warnings. */ \
+ static _PW_TOKENIZER_CONST char PW_CONCAT( \
+ _pw_tokenizer_format_string_, id)[] _PW_TOKENIZER_SECTION(id) = format; \
+ \
+ /* Tokenize the string to a pw_TokenizerStringToken at compile time. */ \
+ _PW_TOKENIZER_CONST pw_TokenizerStringToken _pw_tokenizer_token = \
+ PW_TOKENIZER_STRING_TOKEN(format)
+
+// clang-format on
+
+#ifdef __cplusplus // use constexpr for C++
+#define _PW_TOKENIZER_CONST constexpr
+#else // use const for C
+#define _PW_TOKENIZER_CONST const
+#endif // __cplusplus
+
+// _PW_TOKENIZER_SECTION places the format string in a special .pw_tokenized.#
+// linker section. Host-side decoding tools read the strings from this section
+// to build a database of tokenized strings.
+//
+// This section should be declared as type INFO so that it is excluded from the
+// final binary. To declare the section, as well as the .tokenizer_info section,
+// used for tokenizer metadata, add the following to the linker
+// script's SECTIONS command:
+//
+// .tokenized 0x00000000 (INFO) :
+// {
+// KEEP(*(.tokenized))
+// KEEP(*(.tokenized.*))
+// }
+//
+// .tokenizer_info 0x00000000 (INFO) :
+// {
+// KEEP(*(.tokenizer_info))
+// }
+//
+// Any address could be used for this section, but it should not map to a real
+// device to avoid confusion. 0x00000000 is a reasonable default. An address
+// such as 0xFF000000 that is outside of the ARMv7m memory map could also be
+// used.
+//
+// A linker script snippet that provides these sections is provided in the file
+// tokenizer_linker_sections.ld. This file may be directly included into
+// existing linker scripts.
+//
+// The tokenized string sections can also be managed without linker script
+// modifications, though this is not recommended. The section can be extracted
+// and removed from the ELF with objcopy:
+//
+// objcopy --only-section .tokenize* <ORIGINAL_ELF> <OUTPUT_ELF>
+// objcopy --remove-section .tokenize* <ORIGINAL_ELF>
+//
+// OUTPUT_ELF will be an ELF with only the tokenized strings, and the original
+// ELF file will have the sections removed.
+//
+// Without the above linker script modifications, the section garbage collection
+// option (--gc-sections) removes the tokenized string sections. To avoid
+// editing the target linker script, a separate metadata ELF can be linked
+// without --gc-sections to preserve the tokenized data.
+#define _PW_TOKENIZER_SECTION(unique) \
+ PW_KEEP_IN_SECTION(PW_STRINGIFY(PW_CONCAT(.tokenized., unique)))
diff --git a/pw_tokenizer/pw_tokenizer_private/argument_types_test.h b/pw_tokenizer/pw_tokenizer_private/argument_types_test.h
new file mode 100644
index 0000000..52ac6c9
--- /dev/null
+++ b/pw_tokenizer/pw_tokenizer_private/argument_types_test.h
@@ -0,0 +1,46 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+#pragma once
+
+#include <stdint.h>
+
+#include "pw_preprocessor/util.h"
+#include "pw_tokenizer/internal/argument_types.h"
+
+PW_EXTERN_C_START
+
+pw_TokenizerArgTypes pw_TestTokenizerNoArgs(void);
+
+pw_TokenizerArgTypes pw_TestTokenizerChar(void);
+pw_TokenizerArgTypes pw_TestTokenizerUint8(void);
+pw_TokenizerArgTypes pw_TestTokenizerUint16(void);
+pw_TokenizerArgTypes pw_TestTokenizerInt32(void);
+pw_TokenizerArgTypes pw_TestTokenizerInt64(void);
+pw_TokenizerArgTypes pw_TestTokenizerUint64(void);
+pw_TokenizerArgTypes pw_TestTokenizerFloat(void);
+pw_TokenizerArgTypes pw_TestTokenizerDouble(void);
+pw_TokenizerArgTypes pw_TestTokenizerString(void);
+pw_TokenizerArgTypes pw_TestTokenizerMutableString(void);
+
+pw_TokenizerArgTypes pw_TestTokenizerIntFloat(void);
+pw_TokenizerArgTypes pw_TestTokenizerUint64Char(void);
+pw_TokenizerArgTypes pw_TestTokenizerStringString(void);
+pw_TokenizerArgTypes pw_TestTokenizerUint16Int(void);
+pw_TokenizerArgTypes pw_TestTokenizerFloatString(void);
+
+pw_TokenizerArgTypes pw_TestTokenizerNull(void);
+pw_TokenizerArgTypes pw_TestTokenizerPointer(void);
+pw_TokenizerArgTypes pw_TestTokenizerPointerPointer(void);
+
+PW_EXTERN_C_END
diff --git a/pw_tokenizer/pw_tokenizer_private/generated_hash_test_cases.h b/pw_tokenizer/pw_tokenizer_private/generated_hash_test_cases.h
new file mode 100644
index 0000000..d7667fb
--- /dev/null
+++ b/pw_tokenizer/pw_tokenizer_private/generated_hash_test_cases.h
@@ -0,0 +1,908 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// AUTOGENERATED - DO NOT EDIT
+//
+// This file was generated by generate_hash_test_data.py.
+// To make changes, update the script and run it to generate new files.
+#pragma once
+
+#include <cstddef>
+#include <cstdint>
+#include <string_view>
+
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_128_hash_macro.h"
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_80_hash_macro.h"
+#include "pw_tokenizer/internal/pw_tokenizer_65599_fixed_length_96_hash_macro.h"
+
+namespace pw::tokenizer {
+
+// Test a series of generated test cases.
+inline constexpr struct {
+ std::string_view string;
+ size_t hash_length;
+ uint32_t python_calculated_hash;
+ uint32_t macro_calculated_hash; // clang-format off
+} kHashTests[] = {
+
+{
+ std::string_view("", 0u),
+ 80u, // fixed hash length
+ UINT32_C(0), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH(""), // macro-calculated hash
+},
+{
+ std::string_view("", 0u),
+ 96u, // fixed hash length
+ UINT32_C(0), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH(""), // macro-calculated hash
+},
+{
+ std::string_view("", 0u),
+ 128u, // fixed hash length
+ UINT32_C(0), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH(""), // macro-calculated hash
+},
+{
+ std::string_view("\xa1", 1u),
+ 80u, // fixed hash length
+ UINT32_C(10561440), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xa1"), // macro-calculated hash
+},
+{
+ std::string_view("\xa1", 1u),
+ 96u, // fixed hash length
+ UINT32_C(10561440), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xa1"), // macro-calculated hash
+},
+{
+ std::string_view("\xa1", 1u),
+ 128u, // fixed hash length
+ UINT32_C(10561440), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xa1"), // macro-calculated hash
+},
+{
+ std::string_view("\xff", 1u),
+ 80u, // fixed hash length
+ UINT32_C(16727746), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xff"), // macro-calculated hash
+},
+{
+ std::string_view("\xff", 1u),
+ 96u, // fixed hash length
+ UINT32_C(16727746), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xff"), // macro-calculated hash
+},
+{
+ std::string_view("\xff", 1u),
+ 128u, // fixed hash length
+ UINT32_C(16727746), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xff"), // macro-calculated hash
+},
+{
+ std::string_view("\x00", 1u),
+ 80u, // fixed hash length
+ UINT32_C(1), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\x00", 1u),
+ 96u, // fixed hash length
+ UINT32_C(1), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\x00", 1u),
+ 128u, // fixed hash length
+ UINT32_C(1), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\x00\x00", 2u),
+ 80u, // fixed hash length
+ UINT32_C(2), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x00\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\x00\x00", 2u),
+ 96u, // fixed hash length
+ UINT32_C(2), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x00\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\x00\x00", 2u),
+ 128u, // fixed hash length
+ UINT32_C(2), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x00\x00"), // macro-calculated hash
+},
+{
+ std::string_view("a", 1u),
+ 80u, // fixed hash length
+ UINT32_C(6363104), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("a"), // macro-calculated hash
+},
+{
+ std::string_view("a", 1u),
+ 96u, // fixed hash length
+ UINT32_C(6363104), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("a"), // macro-calculated hash
+},
+{
+ std::string_view("a", 1u),
+ 128u, // fixed hash length
+ UINT32_C(6363104), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("a"), // macro-calculated hash
+},
+{
+ std::string_view("A", 1u),
+ 80u, // fixed hash length
+ UINT32_C(4263936), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("A"), // macro-calculated hash
+},
+{
+ std::string_view("A", 1u),
+ 96u, // fixed hash length
+ UINT32_C(4263936), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("A"), // macro-calculated hash
+},
+{
+ std::string_view("A", 1u),
+ 128u, // fixed hash length
+ UINT32_C(4263936), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("A"), // macro-calculated hash
+},
+{
+ std::string_view("hello, \"world\"", 14u),
+ 80u, // fixed hash length
+ UINT32_C(3537412730), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("hello, \"world\""), // macro-calculated hash
+},
+{
+ std::string_view("hello, \"world\"", 14u),
+ 96u, // fixed hash length
+ UINT32_C(3537412730), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("hello, \"world\""), // macro-calculated hash
+},
+{
+ std::string_view("hello, \"world\"", 14u),
+ 128u, // fixed hash length
+ UINT32_C(3537412730), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("hello, \"world\""), // macro-calculated hash
+},
+{
+ std::string_view("YOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYO", 200u),
+ 80u, // fixed hash length
+ UINT32_C(2035157304), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("YOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYO"), // macro-calculated hash
+},
+{
+ std::string_view("YOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYO", 200u),
+ 96u, // fixed hash length
+ UINT32_C(4222077672), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("YOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYO"), // macro-calculated hash
+},
+{
+ std::string_view("YOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYO", 200u),
+ 128u, // fixed hash length
+ UINT32_C(255790664), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("YOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYOYO"), // macro-calculated hash
+},
+{
+ std::string_view("4", 1u),
+ 80u, // fixed hash length
+ UINT32_C(3411149), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("4"), // macro-calculated hash
+},
+{
+ std::string_view("4", 1u),
+ 96u, // fixed hash length
+ UINT32_C(3411149), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("4"), // macro-calculated hash
+},
+{
+ std::string_view("4", 1u),
+ 128u, // fixed hash length
+ UINT32_C(3411149), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("4"), // macro-calculated hash
+},
+{
+ std::string_view("\xe0", 1u),
+ 80u, // fixed hash length
+ UINT32_C(14694177), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xe0"), // macro-calculated hash
+},
+{
+ std::string_view("\xe0", 1u),
+ 96u, // fixed hash length
+ UINT32_C(14694177), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xe0"), // macro-calculated hash
+},
+{
+ std::string_view("\xe0", 1u),
+ 128u, // fixed hash length
+ UINT32_C(14694177), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xe0"), // macro-calculated hash
+},
+{
+ std::string_view("\x90\xb9", 2u),
+ 80u, // fixed hash length
+ UINT32_C(1537824683), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x90\xb9"), // macro-calculated hash
+},
+{
+ std::string_view("\x90\xb9", 2u),
+ 96u, // fixed hash length
+ UINT32_C(1537824683), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x90\xb9"), // macro-calculated hash
+},
+{
+ std::string_view("\x90\xb9", 2u),
+ 128u, // fixed hash length
+ UINT32_C(1537824683), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x90\xb9"), // macro-calculated hash
+},
+{
+ std::string_view("\x6a\xe7", 2u),
+ 80u, // fixed hash length
+ UINT32_C(1915361151), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x6a\xe7"), // macro-calculated hash
+},
+{
+ std::string_view("\x6a\xe7", 2u),
+ 96u, // fixed hash length
+ UINT32_C(1915361151), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x6a\xe7"), // macro-calculated hash
+},
+{
+ std::string_view("\x6a\xe7", 2u),
+ 128u, // fixed hash length
+ UINT32_C(1915361151), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x6a\xe7"), // macro-calculated hash
+},
+{
+ std::string_view("dy0", 3u),
+ 80u, // fixed hash length
+ UINT32_C(4114649192), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("dy0"), // macro-calculated hash
+},
+{
+ std::string_view("dy0", 3u),
+ 96u, // fixed hash length
+ UINT32_C(4114649192), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("dy0"), // macro-calculated hash
+},
+{
+ std::string_view("dy0", 3u),
+ 128u, // fixed hash length
+ UINT32_C(4114649192), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("dy0"), // macro-calculated hash
+},
+{
+ std::string_view("\xc4\x18\x32", 3u),
+ 80u, // fixed hash length
+ UINT32_C(585787813), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xc4\x18\x32"), // macro-calculated hash
+},
+{
+ std::string_view("\xc4\x18\x32", 3u),
+ 96u, // fixed hash length
+ UINT32_C(585787813), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xc4\x18\x32"), // macro-calculated hash
+},
+{
+ std::string_view("\xc4\x18\x32", 3u),
+ 128u, // fixed hash length
+ UINT32_C(585787813), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xc4\x18\x32"), // macro-calculated hash
+},
+{
+ std::string_view("\x1c\xfc\x28\x2b", 4u),
+ 80u, // fixed hash length
+ UINT32_C(704109799), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x1c\xfc\x28\x2b"), // macro-calculated hash
+},
+{
+ std::string_view("\x1c\xfc\x28\x2b", 4u),
+ 96u, // fixed hash length
+ UINT32_C(704109799), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x1c\xfc\x28\x2b"), // macro-calculated hash
+},
+{
+ std::string_view("\x1c\xfc\x28\x2b", 4u),
+ 128u, // fixed hash length
+ UINT32_C(704109799), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x1c\xfc\x28\x2b"), // macro-calculated hash
+},
+{
+ std::string_view("\xab\x96\x56\x70", 4u),
+ 80u, // fixed hash length
+ UINT32_C(2738614345), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xab\x96\x56\x70"), // macro-calculated hash
+},
+{
+ std::string_view("\xab\x96\x56\x70", 4u),
+ 96u, // fixed hash length
+ UINT32_C(2738614345), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xab\x96\x56\x70"), // macro-calculated hash
+},
+{
+ std::string_view("\xab\x96\x56\x70", 4u),
+ 128u, // fixed hash length
+ UINT32_C(2738614345), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xab\x96\x56\x70"), // macro-calculated hash
+},
+{
+ std::string_view("\x18\x1e\x6e\x6a\x73", 5u),
+ 80u, // fixed hash length
+ UINT32_C(580554452), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x18\x1e\x6e\x6a\x73"), // macro-calculated hash
+},
+{
+ std::string_view("\x18\x1e\x6e\x6a\x73", 5u),
+ 96u, // fixed hash length
+ UINT32_C(580554452), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x18\x1e\x6e\x6a\x73"), // macro-calculated hash
+},
+{
+ std::string_view("\x18\x1e\x6e\x6a\x73", 5u),
+ 128u, // fixed hash length
+ UINT32_C(580554452), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x18\x1e\x6e\x6a\x73"), // macro-calculated hash
+},
+{
+ std::string_view("\xde\xe5\xdf\x22\x00", 5u),
+ 80u, // fixed hash length
+ UINT32_C(4269181327), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xde\xe5\xdf\x22\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\xde\xe5\xdf\x22\x00", 5u),
+ 96u, // fixed hash length
+ UINT32_C(4269181327), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xde\xe5\xdf\x22\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\xde\xe5\xdf\x22\x00", 5u),
+ 128u, // fixed hash length
+ UINT32_C(4269181327), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xde\xe5\xdf\x22\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\x59\xac\x64\x3b\xc7\x36", 6u),
+ 80u, // fixed hash length
+ UINT32_C(2461849503), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x59\xac\x64\x3b\xc7\x36"), // macro-calculated hash
+},
+{
+ std::string_view("\x59\xac\x64\x3b\xc7\x36", 6u),
+ 96u, // fixed hash length
+ UINT32_C(2461849503), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x59\xac\x64\x3b\xc7\x36"), // macro-calculated hash
+},
+{
+ std::string_view("\x59\xac\x64\x3b\xc7\x36", 6u),
+ 128u, // fixed hash length
+ UINT32_C(2461849503), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x59\xac\x64\x3b\xc7\x36"), // macro-calculated hash
+},
+{
+ std::string_view("\xe1\xef\x87\x8d\xbc\xd7", 6u),
+ 80u, // fixed hash length
+ UINT32_C(2407518645), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xe1\xef\x87\x8d\xbc\xd7"), // macro-calculated hash
+},
+{
+ std::string_view("\xe1\xef\x87\x8d\xbc\xd7", 6u),
+ 96u, // fixed hash length
+ UINT32_C(2407518645), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xe1\xef\x87\x8d\xbc\xd7"), // macro-calculated hash
+},
+{
+ std::string_view("\xe1\xef\x87\x8d\xbc\xd7", 6u),
+ 128u, // fixed hash length
+ UINT32_C(2407518645), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xe1\xef\x87\x8d\xbc\xd7"), // macro-calculated hash
+},
+{
+ std::string_view("\x34\xd8\x3a\xbb\xf1\x0e\x07", 7u),
+ 80u, // fixed hash length
+ UINT32_C(2657240642), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x34\xd8\x3a\xbb\xf1\x0e\x07"), // macro-calculated hash
+},
+{
+ std::string_view("\x34\xd8\x3a\xbb\xf1\x0e\x07", 7u),
+ 96u, // fixed hash length
+ UINT32_C(2657240642), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x34\xd8\x3a\xbb\xf1\x0e\x07"), // macro-calculated hash
+},
+{
+ std::string_view("\x34\xd8\x3a\xbb\xf1\x0e\x07", 7u),
+ 128u, // fixed hash length
+ UINT32_C(2657240642), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x34\xd8\x3a\xbb\xf1\x0e\x07"), // macro-calculated hash
+},
+{
+ std::string_view("\xa2\x8e\xb6\x56\x83\xd2\x89", 7u),
+ 80u, // fixed hash length
+ UINT32_C(2016713689), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xa2\x8e\xb6\x56\x83\xd2\x89"), // macro-calculated hash
+},
+{
+ std::string_view("\xa2\x8e\xb6\x56\x83\xd2\x89", 7u),
+ 96u, // fixed hash length
+ UINT32_C(2016713689), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xa2\x8e\xb6\x56\x83\xd2\x89"), // macro-calculated hash
+},
+{
+ std::string_view("\xa2\x8e\xb6\x56\x83\xd2\x89", 7u),
+ 128u, // fixed hash length
+ UINT32_C(2016713689), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xa2\x8e\xb6\x56\x83\xd2\x89"), // macro-calculated hash
+},
+{
+ std::string_view("\x20\x3b\x66\x3f\x80\x8b\xd6\x9f", 8u),
+ 80u, // fixed hash length
+ UINT32_C(727179216), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x20\x3b\x66\x3f\x80\x8b\xd6\x9f"), // macro-calculated hash
+},
+{
+ std::string_view("\x20\x3b\x66\x3f\x80\x8b\xd6\x9f", 8u),
+ 96u, // fixed hash length
+ UINT32_C(727179216), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x20\x3b\x66\x3f\x80\x8b\xd6\x9f"), // macro-calculated hash
+},
+{
+ std::string_view("\x20\x3b\x66\x3f\x80\x8b\xd6\x9f", 8u),
+ 128u, // fixed hash length
+ UINT32_C(727179216), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x20\x3b\x66\x3f\x80\x8b\xd6\x9f"), // macro-calculated hash
+},
+{
+ std::string_view("\xe5\x15\xbf\x96\x52\xd8\x22\x72", 8u),
+ 80u, // fixed hash length
+ UINT32_C(110264805), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xe5\x15\xbf\x96\x52\xd8\x22\x72"), // macro-calculated hash
+},
+{
+ std::string_view("\xe5\x15\xbf\x96\x52\xd8\x22\x72", 8u),
+ 96u, // fixed hash length
+ UINT32_C(110264805), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xe5\x15\xbf\x96\x52\xd8\x22\x72"), // macro-calculated hash
+},
+{
+ std::string_view("\xe5\x15\xbf\x96\x52\xd8\x22\x72", 8u),
+ 128u, // fixed hash length
+ UINT32_C(110264805), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xe5\x15\xbf\x96\x52\xd8\x22\x72"), // macro-calculated hash
+},
+{
+ std::string_view("\x21\x5a\x75\x73\xf1\x70\xc1\x0e\x82", 9u),
+ 80u, // fixed hash length
+ UINT32_C(261914122), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x21\x5a\x75\x73\xf1\x70\xc1\x0e\x82"), // macro-calculated hash
+},
+{
+ std::string_view("\x21\x5a\x75\x73\xf1\x70\xc1\x0e\x82", 9u),
+ 96u, // fixed hash length
+ UINT32_C(261914122), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x21\x5a\x75\x73\xf1\x70\xc1\x0e\x82"), // macro-calculated hash
+},
+{
+ std::string_view("\x21\x5a\x75\x73\xf1\x70\xc1\x0e\x82", 9u),
+ 128u, // fixed hash length
+ UINT32_C(261914122), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x21\x5a\x75\x73\xf1\x70\xc1\x0e\x82"), // macro-calculated hash
+},
+{
+ std::string_view("\x37\x1b\xf3\x87\x5c\xd9\x94\xc6\x40", 9u),
+ 80u, // fixed hash length
+ UINT32_C(1833718768), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x37\x1b\xf3\x87\x5c\xd9\x94\xc6\x40"), // macro-calculated hash
+},
+{
+ std::string_view("\x37\x1b\xf3\x87\x5c\xd9\x94\xc6\x40", 9u),
+ 96u, // fixed hash length
+ UINT32_C(1833718768), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x37\x1b\xf3\x87\x5c\xd9\x94\xc6\x40"), // macro-calculated hash
+},
+{
+ std::string_view("\x37\x1b\xf3\x87\x5c\xd9\x94\xc6\x40", 9u),
+ 128u, // fixed hash length
+ UINT32_C(1833718768), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x37\x1b\xf3\x87\x5c\xd9\x94\xc6\x40"), // macro-calculated hash
+},
+{
+ std::string_view("\x71\x48\x39\xc6\x53\x98\xfa\xc6\x54\x3d", 10u),
+ 80u, // fixed hash length
+ UINT32_C(2326646568), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x71\x48\x39\xc6\x53\x98\xfa\xc6\x54\x3d"), // macro-calculated hash
+},
+{
+ std::string_view("\x71\x48\x39\xc6\x53\x98\xfa\xc6\x54\x3d", 10u),
+ 96u, // fixed hash length
+ UINT32_C(2326646568), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x71\x48\x39\xc6\x53\x98\xfa\xc6\x54\x3d"), // macro-calculated hash
+},
+{
+ std::string_view("\x71\x48\x39\xc6\x53\x98\xfa\xc6\x54\x3d", 10u),
+ 128u, // fixed hash length
+ UINT32_C(2326646568), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x71\x48\x39\xc6\x53\x98\xfa\xc6\x54\x3d"), // macro-calculated hash
+},
+{
+ std::string_view("\x82\x26\x3a\x43\x83\xcf\x86\x3d\x3b\xf5", 10u),
+ 80u, // fixed hash length
+ UINT32_C(2712532084), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x82\x26\x3a\x43\x83\xcf\x86\x3d\x3b\xf5"), // macro-calculated hash
+},
+{
+ std::string_view("\x82\x26\x3a\x43\x83\xcf\x86\x3d\x3b\xf5", 10u),
+ 96u, // fixed hash length
+ UINT32_C(2712532084), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x82\x26\x3a\x43\x83\xcf\x86\x3d\x3b\xf5"), // macro-calculated hash
+},
+{
+ std::string_view("\x82\x26\x3a\x43\x83\xcf\x86\x3d\x3b\xf5", 10u),
+ 128u, // fixed hash length
+ UINT32_C(2712532084), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x82\x26\x3a\x43\x83\xcf\x86\x3d\x3b\xf5"), // macro-calculated hash
+},
+{
+ std::string_view("\xde\x35\x78\x6e\x3f\x98\x61\x43\x53\x28\x24", 11u),
+ 80u, // fixed hash length
+ UINT32_C(544632964), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xde\x35\x78\x6e\x3f\x98\x61\x43\x53\x28\x24"), // macro-calculated hash
+},
+{
+ std::string_view("\xde\x35\x78\x6e\x3f\x98\x61\x43\x53\x28\x24", 11u),
+ 96u, // fixed hash length
+ UINT32_C(544632964), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xde\x35\x78\x6e\x3f\x98\x61\x43\x53\x28\x24"), // macro-calculated hash
+},
+{
+ std::string_view("\xde\x35\x78\x6e\x3f\x98\x61\x43\x53\x28\x24", 11u),
+ 128u, // fixed hash length
+ UINT32_C(544632964), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xde\x35\x78\x6e\x3f\x98\x61\x43\x53\x28\x24"), // macro-calculated hash
+},
+{
+ std::string_view("\x28\x5a\xef\x49\x5c\xfb\x43\x91\xdd\x27\x00", 11u),
+ 80u, // fixed hash length
+ UINT32_C(3878380686), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x28\x5a\xef\x49\x5c\xfb\x43\x91\xdd\x27\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\x28\x5a\xef\x49\x5c\xfb\x43\x91\xdd\x27\x00", 11u),
+ 96u, // fixed hash length
+ UINT32_C(3878380686), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x28\x5a\xef\x49\x5c\xfb\x43\x91\xdd\x27\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\x28\x5a\xef\x49\x5c\xfb\x43\x91\xdd\x27\x00", 11u),
+ 128u, // fixed hash length
+ UINT32_C(3878380686), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x28\x5a\xef\x49\x5c\xfb\x43\x91\xdd\x27\x00"), // macro-calculated hash
+},
+{
+ std::string_view("\xca\x45\x01\x88\x5d\xf2\x24\xa9\x78\xbf\x91\x97", 12u),
+ 80u, // fixed hash length
+ UINT32_C(4053891765), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xca\x45\x01\x88\x5d\xf2\x24\xa9\x78\xbf\x91\x97"), // macro-calculated hash
+},
+{
+ std::string_view("\xca\x45\x01\x88\x5d\xf2\x24\xa9\x78\xbf\x91\x97", 12u),
+ 96u, // fixed hash length
+ UINT32_C(4053891765), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xca\x45\x01\x88\x5d\xf2\x24\xa9\x78\xbf\x91\x97"), // macro-calculated hash
+},
+{
+ std::string_view("\xca\x45\x01\x88\x5d\xf2\x24\xa9\x78\xbf\x91\x97", 12u),
+ 128u, // fixed hash length
+ UINT32_C(4053891765), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xca\x45\x01\x88\x5d\xf2\x24\xa9\x78\xbf\x91\x97"), // macro-calculated hash
+},
+{
+ std::string_view("\x43\xa1\xfb\x5c\x60\x89\xaf\x2b\xdb\xa9\xe5\x59", 12u),
+ 80u, // fixed hash length
+ UINT32_C(2009683698), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x43\xa1\xfb\x5c\x60\x89\xaf\x2b\xdb\xa9\xe5\x59"), // macro-calculated hash
+},
+{
+ std::string_view("\x43\xa1\xfb\x5c\x60\x89\xaf\x2b\xdb\xa9\xe5\x59", 12u),
+ 96u, // fixed hash length
+ UINT32_C(2009683698), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x43\xa1\xfb\x5c\x60\x89\xaf\x2b\xdb\xa9\xe5\x59"), // macro-calculated hash
+},
+{
+ std::string_view("\x43\xa1\xfb\x5c\x60\x89\xaf\x2b\xdb\xa9\xe5\x59", 12u),
+ 128u, // fixed hash length
+ UINT32_C(2009683698), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x43\xa1\xfb\x5c\x60\x89\xaf\x2b\xdb\xa9\xe5\x59"), // macro-calculated hash
+},
+{
+ std::string_view("\xde\x1a\x80\x72\x19\x63\x71\x85\x6c\x53\x51\x7a\x26", 13u),
+ 80u, // fixed hash length
+ UINT32_C(3862326851), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xde\x1a\x80\x72\x19\x63\x71\x85\x6c\x53\x51\x7a\x26"), // macro-calculated hash
+},
+{
+ std::string_view("\xde\x1a\x80\x72\x19\x63\x71\x85\x6c\x53\x51\x7a\x26", 13u),
+ 96u, // fixed hash length
+ UINT32_C(3862326851), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xde\x1a\x80\x72\x19\x63\x71\x85\x6c\x53\x51\x7a\x26"), // macro-calculated hash
+},
+{
+ std::string_view("\xde\x1a\x80\x72\x19\x63\x71\x85\x6c\x53\x51\x7a\x26", 13u),
+ 128u, // fixed hash length
+ UINT32_C(3862326851), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xde\x1a\x80\x72\x19\x63\x71\x85\x6c\x53\x51\x7a\x26"), // macro-calculated hash
+},
+{
+ std::string_view("\x59\x74\xd1\xa5\x70\x0b\xef\x7d\x45\xa9\xcc\xef\x1e", 13u),
+ 80u, // fixed hash length
+ UINT32_C(2358079886), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x59\x74\xd1\xa5\x70\x0b\xef\x7d\x45\xa9\xcc\xef\x1e"), // macro-calculated hash
+},
+{
+ std::string_view("\x59\x74\xd1\xa5\x70\x0b\xef\x7d\x45\xa9\xcc\xef\x1e", 13u),
+ 96u, // fixed hash length
+ UINT32_C(2358079886), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x59\x74\xd1\xa5\x70\x0b\xef\x7d\x45\xa9\xcc\xef\x1e"), // macro-calculated hash
+},
+{
+ std::string_view("\x59\x74\xd1\xa5\x70\x0b\xef\x7d\x45\xa9\xcc\xef\x1e", 13u),
+ 128u, // fixed hash length
+ UINT32_C(2358079886), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x59\x74\xd1\xa5\x70\x0b\xef\x7d\x45\xa9\xcc\xef\x1e"), // macro-calculated hash
+},
+{
+ std::string_view("\xed\xf2\x0a\x96\x1e\xec\x9e\xda\x71\xba\x60\x4d\x49\x8a", 14u),
+ 80u, // fixed hash length
+ UINT32_C(4215296608), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xed\xf2\x0a\x96\x1e\xec\x9e\xda\x71\xba\x60\x4d\x49\x8a"), // macro-calculated hash
+},
+{
+ std::string_view("\xed\xf2\x0a\x96\x1e\xec\x9e\xda\x71\xba\x60\x4d\x49\x8a", 14u),
+ 96u, // fixed hash length
+ UINT32_C(4215296608), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xed\xf2\x0a\x96\x1e\xec\x9e\xda\x71\xba\x60\x4d\x49\x8a"), // macro-calculated hash
+},
+{
+ std::string_view("\xed\xf2\x0a\x96\x1e\xec\x9e\xda\x71\xba\x60\x4d\x49\x8a", 14u),
+ 128u, // fixed hash length
+ UINT32_C(4215296608), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xed\xf2\x0a\x96\x1e\xec\x9e\xda\x71\xba\x60\x4d\x49\x8a"), // macro-calculated hash
+},
+{
+ std::string_view("\x89\x5f\xe4\x0a\xfb\x75\xff\x6a\x24\x1c\x06\xec\xad\xc8", 14u),
+ 80u, // fixed hash length
+ UINT32_C(1051337960), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x89\x5f\xe4\x0a\xfb\x75\xff\x6a\x24\x1c\x06\xec\xad\xc8"), // macro-calculated hash
+},
+{
+ std::string_view("\x89\x5f\xe4\x0a\xfb\x75\xff\x6a\x24\x1c\x06\xec\xad\xc8", 14u),
+ 96u, // fixed hash length
+ UINT32_C(1051337960), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x89\x5f\xe4\x0a\xfb\x75\xff\x6a\x24\x1c\x06\xec\xad\xc8"), // macro-calculated hash
+},
+{
+ std::string_view("\x89\x5f\xe4\x0a\xfb\x75\xff\x6a\x24\x1c\x06\xec\xad\xc8", 14u),
+ 128u, // fixed hash length
+ UINT32_C(1051337960), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x89\x5f\xe4\x0a\xfb\x75\xff\x6a\x24\x1c\x06\xec\xad\xc8"), // macro-calculated hash
+},
+{
+ std::string_view("\x63\xe4\xd0\xdd\xf5\x83\xdb\xa4\x6b\x25\xc2\x2f\x8e\xfe\x1a", 15u),
+ 80u, // fixed hash length
+ UINT32_C(3916582129), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x63\xe4\xd0\xdd\xf5\x83\xdb\xa4\x6b\x25\xc2\x2f\x8e\xfe\x1a"), // macro-calculated hash
+},
+{
+ std::string_view("\x63\xe4\xd0\xdd\xf5\x83\xdb\xa4\x6b\x25\xc2\x2f\x8e\xfe\x1a", 15u),
+ 96u, // fixed hash length
+ UINT32_C(3916582129), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x63\xe4\xd0\xdd\xf5\x83\xdb\xa4\x6b\x25\xc2\x2f\x8e\xfe\x1a"), // macro-calculated hash
+},
+{
+ std::string_view("\x63\xe4\xd0\xdd\xf5\x83\xdb\xa4\x6b\x25\xc2\x2f\x8e\xfe\x1a", 15u),
+ 128u, // fixed hash length
+ UINT32_C(3916582129), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x63\xe4\xd0\xdd\xf5\x83\xdb\xa4\x6b\x25\xc2\x2f\x8e\xfe\x1a"), // macro-calculated hash
+},
+{
+ std::string_view("\x12\xbf\x0e\x44\x14\x1d\x31\x3c\x77\x5e\xf9\xa4\x98\x42\x76", 15u),
+ 80u, // fixed hash length
+ UINT32_C(2665036172), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x12\xbf\x0e\x44\x14\x1d\x31\x3c\x77\x5e\xf9\xa4\x98\x42\x76"), // macro-calculated hash
+},
+{
+ std::string_view("\x12\xbf\x0e\x44\x14\x1d\x31\x3c\x77\x5e\xf9\xa4\x98\x42\x76", 15u),
+ 96u, // fixed hash length
+ UINT32_C(2665036172), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x12\xbf\x0e\x44\x14\x1d\x31\x3c\x77\x5e\xf9\xa4\x98\x42\x76"), // macro-calculated hash
+},
+{
+ std::string_view("\x12\xbf\x0e\x44\x14\x1d\x31\x3c\x77\x5e\xf9\xa4\x98\x42\x76", 15u),
+ 128u, // fixed hash length
+ UINT32_C(2665036172), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x12\xbf\x0e\x44\x14\x1d\x31\x3c\x77\x5e\xf9\xa4\x98\x42\x76"), // macro-calculated hash
+},
+{
+ std::string_view("\x97\xec\x22\xd5\x2d\xdb\xd7\x6a\xd7\x80\xae\xd1\x68\x3a\xca\xbe\x6b\x3c\xbb\x1b\x68\xca\xb4\xde\xa0\xb3\x7f\x80\x44\xd2\xa3\xe9\x80\x56\x06\xb9\xe4\xeb\xde\xe2\x9f\xc5\xcd\xc0\x21\x19\x21\x77\xdc\x38\xf9\x6c\xdb\x00\x64\x46\x40\xfa\x29\xd4\x9c\x87\x3e\x80\xd6\xbe\x4e\xed\x12\x54\xa9\x38\xe3\xff\x6f\x79\x30\xd7\xeb", 79u),
+ 80u, // fixed hash length
+ UINT32_C(2352453932), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x97\xec\x22\xd5\x2d\xdb\xd7\x6a\xd7\x80\xae\xd1\x68\x3a\xca\xbe\x6b\x3c\xbb\x1b\x68\xca\xb4\xde\xa0\xb3\x7f\x80\x44\xd2\xa3\xe9\x80\x56\x06\xb9\xe4\xeb\xde\xe2\x9f\xc5\xcd\xc0\x21\x19\x21\x77\xdc\x38\xf9\x6c\xdb\x00\x64\x46\x40\xfa\x29\xd4\x9c\x87\x3e\x80\xd6\xbe\x4e\xed\x12\x54\xa9\x38\xe3\xff\x6f\x79\x30\xd7\xeb"), // macro-calculated hash
+},
+{
+ std::string_view("\x97\xec\x22\xd5\x2d\xdb\xd7\x6a\xd7\x80\xae\xd1\x68\x3a\xca\xbe\x6b\x3c\xbb\x1b\x68\xca\xb4\xde\xa0\xb3\x7f\x80\x44\xd2\xa3\xe9\x80\x56\x06\xb9\xe4\xeb\xde\xe2\x9f\xc5\xcd\xc0\x21\x19\x21\x77\xdc\x38\xf9\x6c\xdb\x00\x64\x46\x40\xfa\x29\xd4\x9c\x87\x3e\x80\xd6\xbe\x4e\xed\x12\x54\xa9\x38\xe3\xff\x6f\x79\x30\xd7\xeb", 79u),
+ 96u, // fixed hash length
+ UINT32_C(2352453932), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x97\xec\x22\xd5\x2d\xdb\xd7\x6a\xd7\x80\xae\xd1\x68\x3a\xca\xbe\x6b\x3c\xbb\x1b\x68\xca\xb4\xde\xa0\xb3\x7f\x80\x44\xd2\xa3\xe9\x80\x56\x06\xb9\xe4\xeb\xde\xe2\x9f\xc5\xcd\xc0\x21\x19\x21\x77\xdc\x38\xf9\x6c\xdb\x00\x64\x46\x40\xfa\x29\xd4\x9c\x87\x3e\x80\xd6\xbe\x4e\xed\x12\x54\xa9\x38\xe3\xff\x6f\x79\x30\xd7\xeb"), // macro-calculated hash
+},
+{
+ std::string_view("\x97\xec\x22\xd5\x2d\xdb\xd7\x6a\xd7\x80\xae\xd1\x68\x3a\xca\xbe\x6b\x3c\xbb\x1b\x68\xca\xb4\xde\xa0\xb3\x7f\x80\x44\xd2\xa3\xe9\x80\x56\x06\xb9\xe4\xeb\xde\xe2\x9f\xc5\xcd\xc0\x21\x19\x21\x77\xdc\x38\xf9\x6c\xdb\x00\x64\x46\x40\xfa\x29\xd4\x9c\x87\x3e\x80\xd6\xbe\x4e\xed\x12\x54\xa9\x38\xe3\xff\x6f\x79\x30\xd7\xeb", 79u),
+ 128u, // fixed hash length
+ UINT32_C(2352453932), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x97\xec\x22\xd5\x2d\xdb\xd7\x6a\xd7\x80\xae\xd1\x68\x3a\xca\xbe\x6b\x3c\xbb\x1b\x68\xca\xb4\xde\xa0\xb3\x7f\x80\x44\xd2\xa3\xe9\x80\x56\x06\xb9\xe4\xeb\xde\xe2\x9f\xc5\xcd\xc0\x21\x19\x21\x77\xdc\x38\xf9\x6c\xdb\x00\x64\x46\x40\xfa\x29\xd4\x9c\x87\x3e\x80\xd6\xbe\x4e\xed\x12\x54\xa9\x38\xe3\xff\x6f\x79\x30\xd7\xeb"), // macro-calculated hash
+},
+{
+ std::string_view("\x4a\x94\x21\xd0\xe7\xa0\xd4\x23\x23\x38\xc1\x04\xce\xbb\xc9\xe6\xcc\x08\x90\x33\x7b\x0a\x28\x31\xf9\xe3\xe6\xc9\x78\x47\x10\x2c\xda\xb3\xf8\xcd\x7a\x20\xb7\xd1\xd6\x4c\xba\x18\x31\x44\x57\x08\x97\x89\xfd\x43\xce\xf2\x06\x67\xa1\x6d\x15\x47\xa1\xe1\x52\xf6\x4a\x9e\x79\x31\xae\x12\xae\xdd\x4b\x3c\x20\xba\xce\x50\x1e\x13", 80u),
+ 80u, // fixed hash length
+ UINT32_C(4169625832), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x4a\x94\x21\xd0\xe7\xa0\xd4\x23\x23\x38\xc1\x04\xce\xbb\xc9\xe6\xcc\x08\x90\x33\x7b\x0a\x28\x31\xf9\xe3\xe6\xc9\x78\x47\x10\x2c\xda\xb3\xf8\xcd\x7a\x20\xb7\xd1\xd6\x4c\xba\x18\x31\x44\x57\x08\x97\x89\xfd\x43\xce\xf2\x06\x67\xa1\x6d\x15\x47\xa1\xe1\x52\xf6\x4a\x9e\x79\x31\xae\x12\xae\xdd\x4b\x3c\x20\xba\xce\x50\x1e\x13"), // macro-calculated hash
+},
+{
+ std::string_view("\x4a\x94\x21\xd0\xe7\xa0\xd4\x23\x23\x38\xc1\x04\xce\xbb\xc9\xe6\xcc\x08\x90\x33\x7b\x0a\x28\x31\xf9\xe3\xe6\xc9\x78\x47\x10\x2c\xda\xb3\xf8\xcd\x7a\x20\xb7\xd1\xd6\x4c\xba\x18\x31\x44\x57\x08\x97\x89\xfd\x43\xce\xf2\x06\x67\xa1\x6d\x15\x47\xa1\xe1\x52\xf6\x4a\x9e\x79\x31\xae\x12\xae\xdd\x4b\x3c\x20\xba\xce\x50\x1e\x13", 80u),
+ 96u, // fixed hash length
+ UINT32_C(4169625832), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x4a\x94\x21\xd0\xe7\xa0\xd4\x23\x23\x38\xc1\x04\xce\xbb\xc9\xe6\xcc\x08\x90\x33\x7b\x0a\x28\x31\xf9\xe3\xe6\xc9\x78\x47\x10\x2c\xda\xb3\xf8\xcd\x7a\x20\xb7\xd1\xd6\x4c\xba\x18\x31\x44\x57\x08\x97\x89\xfd\x43\xce\xf2\x06\x67\xa1\x6d\x15\x47\xa1\xe1\x52\xf6\x4a\x9e\x79\x31\xae\x12\xae\xdd\x4b\x3c\x20\xba\xce\x50\x1e\x13"), // macro-calculated hash
+},
+{
+ std::string_view("\x4a\x94\x21\xd0\xe7\xa0\xd4\x23\x23\x38\xc1\x04\xce\xbb\xc9\xe6\xcc\x08\x90\x33\x7b\x0a\x28\x31\xf9\xe3\xe6\xc9\x78\x47\x10\x2c\xda\xb3\xf8\xcd\x7a\x20\xb7\xd1\xd6\x4c\xba\x18\x31\x44\x57\x08\x97\x89\xfd\x43\xce\xf2\x06\x67\xa1\x6d\x15\x47\xa1\xe1\x52\xf6\x4a\x9e\x79\x31\xae\x12\xae\xdd\x4b\x3c\x20\xba\xce\x50\x1e\x13", 80u),
+ 128u, // fixed hash length
+ UINT32_C(4169625832), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x4a\x94\x21\xd0\xe7\xa0\xd4\x23\x23\x38\xc1\x04\xce\xbb\xc9\xe6\xcc\x08\x90\x33\x7b\x0a\x28\x31\xf9\xe3\xe6\xc9\x78\x47\x10\x2c\xda\xb3\xf8\xcd\x7a\x20\xb7\xd1\xd6\x4c\xba\x18\x31\x44\x57\x08\x97\x89\xfd\x43\xce\xf2\x06\x67\xa1\x6d\x15\x47\xa1\xe1\x52\xf6\x4a\x9e\x79\x31\xae\x12\xae\xdd\x4b\x3c\x20\xba\xce\x50\x1e\x13"), // macro-calculated hash
+},
+{
+ std::string_view("\x79\x1d\xba\x71\x02\x36\xfd\xaf\xbe\x49\x5e\x0b\x77\x7b\x57\xf7\x8b\xad\x6a\xe3\xc5\x57\x5a\x34\xa6\x12\xb2\xb2\x8a\x4e\x11\x13\xa5\x97\x2f\xf6\xbc\x62\xdb\x63\x0b\xa4\xc3\x3d\x66\x92\x3e\x8b\x53\x47\x12\x3a\x36\x9a\xe2\x31\xf9\x0d\x62\x71\x79\x3f\xa7\x04\x09\x8c\x40\xa7\x8a\x17\x3a\xb3\x6a\xea\x51\xdf\x91\x5a\x1d\x42\x4e", 81u),
+ 80u, // fixed hash length
+ UINT32_C(2417296923), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x79\x1d\xba\x71\x02\x36\xfd\xaf\xbe\x49\x5e\x0b\x77\x7b\x57\xf7\x8b\xad\x6a\xe3\xc5\x57\x5a\x34\xa6\x12\xb2\xb2\x8a\x4e\x11\x13\xa5\x97\x2f\xf6\xbc\x62\xdb\x63\x0b\xa4\xc3\x3d\x66\x92\x3e\x8b\x53\x47\x12\x3a\x36\x9a\xe2\x31\xf9\x0d\x62\x71\x79\x3f\xa7\x04\x09\x8c\x40\xa7\x8a\x17\x3a\xb3\x6a\xea\x51\xdf\x91\x5a\x1d\x42\x4e"), // macro-calculated hash
+},
+{
+ std::string_view("\x79\x1d\xba\x71\x02\x36\xfd\xaf\xbe\x49\x5e\x0b\x77\x7b\x57\xf7\x8b\xad\x6a\xe3\xc5\x57\x5a\x34\xa6\x12\xb2\xb2\x8a\x4e\x11\x13\xa5\x97\x2f\xf6\xbc\x62\xdb\x63\x0b\xa4\xc3\x3d\x66\x92\x3e\x8b\x53\x47\x12\x3a\x36\x9a\xe2\x31\xf9\x0d\x62\x71\x79\x3f\xa7\x04\x09\x8c\x40\xa7\x8a\x17\x3a\xb3\x6a\xea\x51\xdf\x91\x5a\x1d\x42\x4e", 81u),
+ 96u, // fixed hash length
+ UINT32_C(987115853), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x79\x1d\xba\x71\x02\x36\xfd\xaf\xbe\x49\x5e\x0b\x77\x7b\x57\xf7\x8b\xad\x6a\xe3\xc5\x57\x5a\x34\xa6\x12\xb2\xb2\x8a\x4e\x11\x13\xa5\x97\x2f\xf6\xbc\x62\xdb\x63\x0b\xa4\xc3\x3d\x66\x92\x3e\x8b\x53\x47\x12\x3a\x36\x9a\xe2\x31\xf9\x0d\x62\x71\x79\x3f\xa7\x04\x09\x8c\x40\xa7\x8a\x17\x3a\xb3\x6a\xea\x51\xdf\x91\x5a\x1d\x42\x4e"), // macro-calculated hash
+},
+{
+ std::string_view("\x79\x1d\xba\x71\x02\x36\xfd\xaf\xbe\x49\x5e\x0b\x77\x7b\x57\xf7\x8b\xad\x6a\xe3\xc5\x57\x5a\x34\xa6\x12\xb2\xb2\x8a\x4e\x11\x13\xa5\x97\x2f\xf6\xbc\x62\xdb\x63\x0b\xa4\xc3\x3d\x66\x92\x3e\x8b\x53\x47\x12\x3a\x36\x9a\xe2\x31\xf9\x0d\x62\x71\x79\x3f\xa7\x04\x09\x8c\x40\xa7\x8a\x17\x3a\xb3\x6a\xea\x51\xdf\x91\x5a\x1d\x42\x4e", 81u),
+ 128u, // fixed hash length
+ UINT32_C(987115853), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x79\x1d\xba\x71\x02\x36\xfd\xaf\xbe\x49\x5e\x0b\x77\x7b\x57\xf7\x8b\xad\x6a\xe3\xc5\x57\x5a\x34\xa6\x12\xb2\xb2\x8a\x4e\x11\x13\xa5\x97\x2f\xf6\xbc\x62\xdb\x63\x0b\xa4\xc3\x3d\x66\x92\x3e\x8b\x53\x47\x12\x3a\x36\x9a\xe2\x31\xf9\x0d\x62\x71\x79\x3f\xa7\x04\x09\x8c\x40\xa7\x8a\x17\x3a\xb3\x6a\xea\x51\xdf\x91\x5a\x1d\x42\x4e"), // macro-calculated hash
+},
+{
+ std::string_view("\x08\xd5\x5f\x9b\x1a\xd5\x15\x4b\x80\x3f\x01\x35\x6f\xda\xf3\x9a\x2d\x8d\xb6\xb2\x36\x8b\xc4\x69\x46\xfe\xe1\x3f\x83\xbc\x45\xc8\x53\x75\xf5\x89\x22\x8b\x14\xfa\xd0\xce\xc9\x85\xe8\x98\x6b\x47\xc4\xa5\xf9\x06\x4c\x39\xdc\x8c\xe2\xf1\xa4\x59\x1c\xc1\xd4\x16\xb1\xb4\x2a\x61\x2c\x48\x2c\x7f\xd2\x1f\x77\xd2\x92\xf9\xfa\x84\x8c\x74\xc7\xa1\x3d\x72\x46\x97\x63\xc1\x97\x9a\x4b\xb2\x17", 95u),
+ 80u, // fixed hash length
+ UINT32_C(1750895817), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x08\xd5\x5f\x9b\x1a\xd5\x15\x4b\x80\x3f\x01\x35\x6f\xda\xf3\x9a\x2d\x8d\xb6\xb2\x36\x8b\xc4\x69\x46\xfe\xe1\x3f\x83\xbc\x45\xc8\x53\x75\xf5\x89\x22\x8b\x14\xfa\xd0\xce\xc9\x85\xe8\x98\x6b\x47\xc4\xa5\xf9\x06\x4c\x39\xdc\x8c\xe2\xf1\xa4\x59\x1c\xc1\xd4\x16\xb1\xb4\x2a\x61\x2c\x48\x2c\x7f\xd2\x1f\x77\xd2\x92\xf9\xfa\x84\x8c\x74\xc7\xa1\x3d\x72\x46\x97\x63\xc1\x97\x9a\x4b\xb2\x17"), // macro-calculated hash
+},
+{
+ std::string_view("\x08\xd5\x5f\x9b\x1a\xd5\x15\x4b\x80\x3f\x01\x35\x6f\xda\xf3\x9a\x2d\x8d\xb6\xb2\x36\x8b\xc4\x69\x46\xfe\xe1\x3f\x83\xbc\x45\xc8\x53\x75\xf5\x89\x22\x8b\x14\xfa\xd0\xce\xc9\x85\xe8\x98\x6b\x47\xc4\xa5\xf9\x06\x4c\x39\xdc\x8c\xe2\xf1\xa4\x59\x1c\xc1\xd4\x16\xb1\xb4\x2a\x61\x2c\x48\x2c\x7f\xd2\x1f\x77\xd2\x92\xf9\xfa\x84\x8c\x74\xc7\xa1\x3d\x72\x46\x97\x63\xc1\x97\x9a\x4b\xb2\x17", 95u),
+ 96u, // fixed hash length
+ UINT32_C(720276802), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x08\xd5\x5f\x9b\x1a\xd5\x15\x4b\x80\x3f\x01\x35\x6f\xda\xf3\x9a\x2d\x8d\xb6\xb2\x36\x8b\xc4\x69\x46\xfe\xe1\x3f\x83\xbc\x45\xc8\x53\x75\xf5\x89\x22\x8b\x14\xfa\xd0\xce\xc9\x85\xe8\x98\x6b\x47\xc4\xa5\xf9\x06\x4c\x39\xdc\x8c\xe2\xf1\xa4\x59\x1c\xc1\xd4\x16\xb1\xb4\x2a\x61\x2c\x48\x2c\x7f\xd2\x1f\x77\xd2\x92\xf9\xfa\x84\x8c\x74\xc7\xa1\x3d\x72\x46\x97\x63\xc1\x97\x9a\x4b\xb2\x17"), // macro-calculated hash
+},
+{
+ std::string_view("\x08\xd5\x5f\x9b\x1a\xd5\x15\x4b\x80\x3f\x01\x35\x6f\xda\xf3\x9a\x2d\x8d\xb6\xb2\x36\x8b\xc4\x69\x46\xfe\xe1\x3f\x83\xbc\x45\xc8\x53\x75\xf5\x89\x22\x8b\x14\xfa\xd0\xce\xc9\x85\xe8\x98\x6b\x47\xc4\xa5\xf9\x06\x4c\x39\xdc\x8c\xe2\xf1\xa4\x59\x1c\xc1\xd4\x16\xb1\xb4\x2a\x61\x2c\x48\x2c\x7f\xd2\x1f\x77\xd2\x92\xf9\xfa\x84\x8c\x74\xc7\xa1\x3d\x72\x46\x97\x63\xc1\x97\x9a\x4b\xb2\x17", 95u),
+ 128u, // fixed hash length
+ UINT32_C(720276802), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x08\xd5\x5f\x9b\x1a\xd5\x15\x4b\x80\x3f\x01\x35\x6f\xda\xf3\x9a\x2d\x8d\xb6\xb2\x36\x8b\xc4\x69\x46\xfe\xe1\x3f\x83\xbc\x45\xc8\x53\x75\xf5\x89\x22\x8b\x14\xfa\xd0\xce\xc9\x85\xe8\x98\x6b\x47\xc4\xa5\xf9\x06\x4c\x39\xdc\x8c\xe2\xf1\xa4\x59\x1c\xc1\xd4\x16\xb1\xb4\x2a\x61\x2c\x48\x2c\x7f\xd2\x1f\x77\xd2\x92\xf9\xfa\x84\x8c\x74\xc7\xa1\x3d\x72\x46\x97\x63\xc1\x97\x9a\x4b\xb2\x17"), // macro-calculated hash
+},
+{
+ std::string_view("\x9b\xf2\x2d\xc5\x5f\xe7\xa6\xf3\xf6\xd8\x2c\x7f\x89\x72\x1d\xba\x88\x1a\x84\xb1\x7b\xad\x24\x96\x31\x80\x10\x2b\x1f\x32\x06\xc8\xef\x00\x5a\xe2\x9c\xfc\x3a\x6f\x5d\x70\xc0\x06\xe0\x8b\xcd\xd5\xec\xf4\x25\x91\xd9\xe4\x86\x4f\x3a\xdb\x36\x42\xde\x57\x8d\x5b\xeb\xd3\x67\x47\x99\x0b\x1b\x26\xd1\x06\x93\x5e\xa2\xf9\xc3\x28\x2e\x51\xed\x99\x12\x84\xd8\x79\x85\x12\x16\xde\x1d\xdc\x47\x4b", 96u),
+ 80u, // fixed hash length
+ UINT32_C(760136888), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x9b\xf2\x2d\xc5\x5f\xe7\xa6\xf3\xf6\xd8\x2c\x7f\x89\x72\x1d\xba\x88\x1a\x84\xb1\x7b\xad\x24\x96\x31\x80\x10\x2b\x1f\x32\x06\xc8\xef\x00\x5a\xe2\x9c\xfc\x3a\x6f\x5d\x70\xc0\x06\xe0\x8b\xcd\xd5\xec\xf4\x25\x91\xd9\xe4\x86\x4f\x3a\xdb\x36\x42\xde\x57\x8d\x5b\xeb\xd3\x67\x47\x99\x0b\x1b\x26\xd1\x06\x93\x5e\xa2\xf9\xc3\x28\x2e\x51\xed\x99\x12\x84\xd8\x79\x85\x12\x16\xde\x1d\xdc\x47\x4b"), // macro-calculated hash
+},
+{
+ std::string_view("\x9b\xf2\x2d\xc5\x5f\xe7\xa6\xf3\xf6\xd8\x2c\x7f\x89\x72\x1d\xba\x88\x1a\x84\xb1\x7b\xad\x24\x96\x31\x80\x10\x2b\x1f\x32\x06\xc8\xef\x00\x5a\xe2\x9c\xfc\x3a\x6f\x5d\x70\xc0\x06\xe0\x8b\xcd\xd5\xec\xf4\x25\x91\xd9\xe4\x86\x4f\x3a\xdb\x36\x42\xde\x57\x8d\x5b\xeb\xd3\x67\x47\x99\x0b\x1b\x26\xd1\x06\x93\x5e\xa2\xf9\xc3\x28\x2e\x51\xed\x99\x12\x84\xd8\x79\x85\x12\x16\xde\x1d\xdc\x47\x4b", 96u),
+ 96u, // fixed hash length
+ UINT32_C(1408671026), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x9b\xf2\x2d\xc5\x5f\xe7\xa6\xf3\xf6\xd8\x2c\x7f\x89\x72\x1d\xba\x88\x1a\x84\xb1\x7b\xad\x24\x96\x31\x80\x10\x2b\x1f\x32\x06\xc8\xef\x00\x5a\xe2\x9c\xfc\x3a\x6f\x5d\x70\xc0\x06\xe0\x8b\xcd\xd5\xec\xf4\x25\x91\xd9\xe4\x86\x4f\x3a\xdb\x36\x42\xde\x57\x8d\x5b\xeb\xd3\x67\x47\x99\x0b\x1b\x26\xd1\x06\x93\x5e\xa2\xf9\xc3\x28\x2e\x51\xed\x99\x12\x84\xd8\x79\x85\x12\x16\xde\x1d\xdc\x47\x4b"), // macro-calculated hash
+},
+{
+ std::string_view("\x9b\xf2\x2d\xc5\x5f\xe7\xa6\xf3\xf6\xd8\x2c\x7f\x89\x72\x1d\xba\x88\x1a\x84\xb1\x7b\xad\x24\x96\x31\x80\x10\x2b\x1f\x32\x06\xc8\xef\x00\x5a\xe2\x9c\xfc\x3a\x6f\x5d\x70\xc0\x06\xe0\x8b\xcd\xd5\xec\xf4\x25\x91\xd9\xe4\x86\x4f\x3a\xdb\x36\x42\xde\x57\x8d\x5b\xeb\xd3\x67\x47\x99\x0b\x1b\x26\xd1\x06\x93\x5e\xa2\xf9\xc3\x28\x2e\x51\xed\x99\x12\x84\xd8\x79\x85\x12\x16\xde\x1d\xdc\x47\x4b", 96u),
+ 128u, // fixed hash length
+ UINT32_C(1408671026), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x9b\xf2\x2d\xc5\x5f\xe7\xa6\xf3\xf6\xd8\x2c\x7f\x89\x72\x1d\xba\x88\x1a\x84\xb1\x7b\xad\x24\x96\x31\x80\x10\x2b\x1f\x32\x06\xc8\xef\x00\x5a\xe2\x9c\xfc\x3a\x6f\x5d\x70\xc0\x06\xe0\x8b\xcd\xd5\xec\xf4\x25\x91\xd9\xe4\x86\x4f\x3a\xdb\x36\x42\xde\x57\x8d\x5b\xeb\xd3\x67\x47\x99\x0b\x1b\x26\xd1\x06\x93\x5e\xa2\xf9\xc3\x28\x2e\x51\xed\x99\x12\x84\xd8\x79\x85\x12\x16\xde\x1d\xdc\x47\x4b"), // macro-calculated hash
+},
+{
+ std::string_view("\xa7\x97\xb4\x6c\x4b\x3e\xa9\x40\x2d\x1c\x46\xd6\x42\xf7\xee\xd3\xc4\xa7\xa8\xbd\xd3\xe0\x1a\x56\x31\x4e\x9b\xbd\x28\x16\x56\x1f\x38\x57\x56\x8b\x7d\xa4\xc9\xe4\xb2\xce\x3c\xf8\x0f\x13\x83\x35\x66\x86\xdf\x33\xfa\x6e\x09\xf7\x3c\x05\xd9\x05\xb3\xb6\x62\xc2\xd9\x75\x00\x7f\x00\xd9\x2c\x67\x78\x8c\x4c\x45\x3b\x9b\xc7\xaf\x6e\xdf\x23\x79\x09\xa4\xbb\x29\x29\x64\xd4\xc7\x2c\x50\x83\x24\xc7", 97u),
+ 80u, // fixed hash length
+ UINT32_C(4113347769), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xa7\x97\xb4\x6c\x4b\x3e\xa9\x40\x2d\x1c\x46\xd6\x42\xf7\xee\xd3\xc4\xa7\xa8\xbd\xd3\xe0\x1a\x56\x31\x4e\x9b\xbd\x28\x16\x56\x1f\x38\x57\x56\x8b\x7d\xa4\xc9\xe4\xb2\xce\x3c\xf8\x0f\x13\x83\x35\x66\x86\xdf\x33\xfa\x6e\x09\xf7\x3c\x05\xd9\x05\xb3\xb6\x62\xc2\xd9\x75\x00\x7f\x00\xd9\x2c\x67\x78\x8c\x4c\x45\x3b\x9b\xc7\xaf\x6e\xdf\x23\x79\x09\xa4\xbb\x29\x29\x64\xd4\xc7\x2c\x50\x83\x24\xc7"), // macro-calculated hash
+},
+{
+ std::string_view("\xa7\x97\xb4\x6c\x4b\x3e\xa9\x40\x2d\x1c\x46\xd6\x42\xf7\xee\xd3\xc4\xa7\xa8\xbd\xd3\xe0\x1a\x56\x31\x4e\x9b\xbd\x28\x16\x56\x1f\x38\x57\x56\x8b\x7d\xa4\xc9\xe4\xb2\xce\x3c\xf8\x0f\x13\x83\x35\x66\x86\xdf\x33\xfa\x6e\x09\xf7\x3c\x05\xd9\x05\xb3\xb6\x62\xc2\xd9\x75\x00\x7f\x00\xd9\x2c\x67\x78\x8c\x4c\x45\x3b\x9b\xc7\xaf\x6e\xdf\x23\x79\x09\xa4\xbb\x29\x29\x64\xd4\xc7\x2c\x50\x83\x24\xc7", 97u),
+ 96u, // fixed hash length
+ UINT32_C(1367119804), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xa7\x97\xb4\x6c\x4b\x3e\xa9\x40\x2d\x1c\x46\xd6\x42\xf7\xee\xd3\xc4\xa7\xa8\xbd\xd3\xe0\x1a\x56\x31\x4e\x9b\xbd\x28\x16\x56\x1f\x38\x57\x56\x8b\x7d\xa4\xc9\xe4\xb2\xce\x3c\xf8\x0f\x13\x83\x35\x66\x86\xdf\x33\xfa\x6e\x09\xf7\x3c\x05\xd9\x05\xb3\xb6\x62\xc2\xd9\x75\x00\x7f\x00\xd9\x2c\x67\x78\x8c\x4c\x45\x3b\x9b\xc7\xaf\x6e\xdf\x23\x79\x09\xa4\xbb\x29\x29\x64\xd4\xc7\x2c\x50\x83\x24\xc7"), // macro-calculated hash
+},
+{
+ std::string_view("\xa7\x97\xb4\x6c\x4b\x3e\xa9\x40\x2d\x1c\x46\xd6\x42\xf7\xee\xd3\xc4\xa7\xa8\xbd\xd3\xe0\x1a\x56\x31\x4e\x9b\xbd\x28\x16\x56\x1f\x38\x57\x56\x8b\x7d\xa4\xc9\xe4\xb2\xce\x3c\xf8\x0f\x13\x83\x35\x66\x86\xdf\x33\xfa\x6e\x09\xf7\x3c\x05\xd9\x05\xb3\xb6\x62\xc2\xd9\x75\x00\x7f\x00\xd9\x2c\x67\x78\x8c\x4c\x45\x3b\x9b\xc7\xaf\x6e\xdf\x23\x79\x09\xa4\xbb\x29\x29\x64\xd4\xc7\x2c\x50\x83\x24\xc7", 97u),
+ 128u, // fixed hash length
+ UINT32_C(687960245), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xa7\x97\xb4\x6c\x4b\x3e\xa9\x40\x2d\x1c\x46\xd6\x42\xf7\xee\xd3\xc4\xa7\xa8\xbd\xd3\xe0\x1a\x56\x31\x4e\x9b\xbd\x28\x16\x56\x1f\x38\x57\x56\x8b\x7d\xa4\xc9\xe4\xb2\xce\x3c\xf8\x0f\x13\x83\x35\x66\x86\xdf\x33\xfa\x6e\x09\xf7\x3c\x05\xd9\x05\xb3\xb6\x62\xc2\xd9\x75\x00\x7f\x00\xd9\x2c\x67\x78\x8c\x4c\x45\x3b\x9b\xc7\xaf\x6e\xdf\x23\x79\x09\xa4\xbb\x29\x29\x64\xd4\xc7\x2c\x50\x83\x24\xc7"), // macro-calculated hash
+},
+{
+ std::string_view("\xf1\xdf\xb7\x7c\xcb\xf7\xc8\xe8\x94\xd3\x04\x33\x3e\x1a\x9c\x7d\x4a\xa4\xc3\xaf\x87\x97\xdb\xcc\xd6\x23\xe5\x0d\xf2\xa9\x9e\x63\xf6\xc4\xa8\x4d\x56\x86\x2b\x9c\x5f\xca\x4c\x17\x88\xa4\x5c\x73\x80\x6b\x2c\x03\x0e\xcd\x9f\xb8\x99\x44\x40\x2a\x33\x73\x94\xd1\x3f\x64\x56\x76\xf1\x9f\xfc\xb4\x1b\x1c\xa7\xc8\x28\x90\x03\x21\xe1\xcf\xb4\x57\x14\x2b\x5f\xbb\x84\x99\x4f\x16\xd4\x25\xe4\x54\x6f\xcd\x2c\x0f\x70\x98\xdb\xf3\xb0\xfe\xe0\x1a\x2e\x3d\x8b\x26\xdf\x28\x41\x16\x21\xc4\x86\x43\x9a\x29\x2b\xa3\x18\x74\x98", 127u),
+ 80u, // fixed hash length
+ UINT32_C(1288060573), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\xf1\xdf\xb7\x7c\xcb\xf7\xc8\xe8\x94\xd3\x04\x33\x3e\x1a\x9c\x7d\x4a\xa4\xc3\xaf\x87\x97\xdb\xcc\xd6\x23\xe5\x0d\xf2\xa9\x9e\x63\xf6\xc4\xa8\x4d\x56\x86\x2b\x9c\x5f\xca\x4c\x17\x88\xa4\x5c\x73\x80\x6b\x2c\x03\x0e\xcd\x9f\xb8\x99\x44\x40\x2a\x33\x73\x94\xd1\x3f\x64\x56\x76\xf1\x9f\xfc\xb4\x1b\x1c\xa7\xc8\x28\x90\x03\x21\xe1\xcf\xb4\x57\x14\x2b\x5f\xbb\x84\x99\x4f\x16\xd4\x25\xe4\x54\x6f\xcd\x2c\x0f\x70\x98\xdb\xf3\xb0\xfe\xe0\x1a\x2e\x3d\x8b\x26\xdf\x28\x41\x16\x21\xc4\x86\x43\x9a\x29\x2b\xa3\x18\x74\x98"), // macro-calculated hash
+},
+{
+ std::string_view("\xf1\xdf\xb7\x7c\xcb\xf7\xc8\xe8\x94\xd3\x04\x33\x3e\x1a\x9c\x7d\x4a\xa4\xc3\xaf\x87\x97\xdb\xcc\xd6\x23\xe5\x0d\xf2\xa9\x9e\x63\xf6\xc4\xa8\x4d\x56\x86\x2b\x9c\x5f\xca\x4c\x17\x88\xa4\x5c\x73\x80\x6b\x2c\x03\x0e\xcd\x9f\xb8\x99\x44\x40\x2a\x33\x73\x94\xd1\x3f\x64\x56\x76\xf1\x9f\xfc\xb4\x1b\x1c\xa7\xc8\x28\x90\x03\x21\xe1\xcf\xb4\x57\x14\x2b\x5f\xbb\x84\x99\x4f\x16\xd4\x25\xe4\x54\x6f\xcd\x2c\x0f\x70\x98\xdb\xf3\xb0\xfe\xe0\x1a\x2e\x3d\x8b\x26\xdf\x28\x41\x16\x21\xc4\x86\x43\x9a\x29\x2b\xa3\x18\x74\x98", 127u),
+ 96u, // fixed hash length
+ UINT32_C(1810369278), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\xf1\xdf\xb7\x7c\xcb\xf7\xc8\xe8\x94\xd3\x04\x33\x3e\x1a\x9c\x7d\x4a\xa4\xc3\xaf\x87\x97\xdb\xcc\xd6\x23\xe5\x0d\xf2\xa9\x9e\x63\xf6\xc4\xa8\x4d\x56\x86\x2b\x9c\x5f\xca\x4c\x17\x88\xa4\x5c\x73\x80\x6b\x2c\x03\x0e\xcd\x9f\xb8\x99\x44\x40\x2a\x33\x73\x94\xd1\x3f\x64\x56\x76\xf1\x9f\xfc\xb4\x1b\x1c\xa7\xc8\x28\x90\x03\x21\xe1\xcf\xb4\x57\x14\x2b\x5f\xbb\x84\x99\x4f\x16\xd4\x25\xe4\x54\x6f\xcd\x2c\x0f\x70\x98\xdb\xf3\xb0\xfe\xe0\x1a\x2e\x3d\x8b\x26\xdf\x28\x41\x16\x21\xc4\x86\x43\x9a\x29\x2b\xa3\x18\x74\x98"), // macro-calculated hash
+},
+{
+ std::string_view("\xf1\xdf\xb7\x7c\xcb\xf7\xc8\xe8\x94\xd3\x04\x33\x3e\x1a\x9c\x7d\x4a\xa4\xc3\xaf\x87\x97\xdb\xcc\xd6\x23\xe5\x0d\xf2\xa9\x9e\x63\xf6\xc4\xa8\x4d\x56\x86\x2b\x9c\x5f\xca\x4c\x17\x88\xa4\x5c\x73\x80\x6b\x2c\x03\x0e\xcd\x9f\xb8\x99\x44\x40\x2a\x33\x73\x94\xd1\x3f\x64\x56\x76\xf1\x9f\xfc\xb4\x1b\x1c\xa7\xc8\x28\x90\x03\x21\xe1\xcf\xb4\x57\x14\x2b\x5f\xbb\x84\x99\x4f\x16\xd4\x25\xe4\x54\x6f\xcd\x2c\x0f\x70\x98\xdb\xf3\xb0\xfe\xe0\x1a\x2e\x3d\x8b\x26\xdf\x28\x41\x16\x21\xc4\x86\x43\x9a\x29\x2b\xa3\x18\x74\x98", 127u),
+ 128u, // fixed hash length
+ UINT32_C(2429195322), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\xf1\xdf\xb7\x7c\xcb\xf7\xc8\xe8\x94\xd3\x04\x33\x3e\x1a\x9c\x7d\x4a\xa4\xc3\xaf\x87\x97\xdb\xcc\xd6\x23\xe5\x0d\xf2\xa9\x9e\x63\xf6\xc4\xa8\x4d\x56\x86\x2b\x9c\x5f\xca\x4c\x17\x88\xa4\x5c\x73\x80\x6b\x2c\x03\x0e\xcd\x9f\xb8\x99\x44\x40\x2a\x33\x73\x94\xd1\x3f\x64\x56\x76\xf1\x9f\xfc\xb4\x1b\x1c\xa7\xc8\x28\x90\x03\x21\xe1\xcf\xb4\x57\x14\x2b\x5f\xbb\x84\x99\x4f\x16\xd4\x25\xe4\x54\x6f\xcd\x2c\x0f\x70\x98\xdb\xf3\xb0\xfe\xe0\x1a\x2e\x3d\x8b\x26\xdf\x28\x41\x16\x21\xc4\x86\x43\x9a\x29\x2b\xa3\x18\x74\x98"), // macro-calculated hash
+},
+{
+ std::string_view("\x43\xd7\x5e\xff\x9b\x29\x34\x97\x50\x92\xb3\xcd\x38\xee\x3c\xdf\xc7\x7a\x76\x4e\x72\xaf\xb2\xdd\x37\x1c\x8e\x1c\xec\x08\x15\x72\x4f\xac\x2d\x67\x19\xc1\xc5\x6f\x52\x7d\x79\xe9\xa8\x3d\xcb\x3b\xdb\x4d\x29\x81\x89\xa8\x1f\xcb\xbf\xc6\x2f\x5a\xab\x9e\x05\xc6\xec\x42\x24\xf0\x9e\x2c\xb4\x7e\xc1\x85\x10\x3c\xc0\xd5\x99\x9a\x62\x52\x04\xde\xe1\xd1\x44\x5e\x4b\x4b\xc1\x10\xf1\xbe\x06\xc8\xf7\x07\xd2\x92\xc8\x92\xb3\xb6\xa5\x79\x6e\x93\x6d\xfa\xd1\x68\x6e\xec\x25\x10\xcf\x4f\x8e\xd4\xfb\x3c\x56\x04\xbc\x6f\xbc\x65", 128u),
+ 80u, // fixed hash length
+ UINT32_C(1125261758), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x43\xd7\x5e\xff\x9b\x29\x34\x97\x50\x92\xb3\xcd\x38\xee\x3c\xdf\xc7\x7a\x76\x4e\x72\xaf\xb2\xdd\x37\x1c\x8e\x1c\xec\x08\x15\x72\x4f\xac\x2d\x67\x19\xc1\xc5\x6f\x52\x7d\x79\xe9\xa8\x3d\xcb\x3b\xdb\x4d\x29\x81\x89\xa8\x1f\xcb\xbf\xc6\x2f\x5a\xab\x9e\x05\xc6\xec\x42\x24\xf0\x9e\x2c\xb4\x7e\xc1\x85\x10\x3c\xc0\xd5\x99\x9a\x62\x52\x04\xde\xe1\xd1\x44\x5e\x4b\x4b\xc1\x10\xf1\xbe\x06\xc8\xf7\x07\xd2\x92\xc8\x92\xb3\xb6\xa5\x79\x6e\x93\x6d\xfa\xd1\x68\x6e\xec\x25\x10\xcf\x4f\x8e\xd4\xfb\x3c\x56\x04\xbc\x6f\xbc\x65"), // macro-calculated hash
+},
+{
+ std::string_view("\x43\xd7\x5e\xff\x9b\x29\x34\x97\x50\x92\xb3\xcd\x38\xee\x3c\xdf\xc7\x7a\x76\x4e\x72\xaf\xb2\xdd\x37\x1c\x8e\x1c\xec\x08\x15\x72\x4f\xac\x2d\x67\x19\xc1\xc5\x6f\x52\x7d\x79\xe9\xa8\x3d\xcb\x3b\xdb\x4d\x29\x81\x89\xa8\x1f\xcb\xbf\xc6\x2f\x5a\xab\x9e\x05\xc6\xec\x42\x24\xf0\x9e\x2c\xb4\x7e\xc1\x85\x10\x3c\xc0\xd5\x99\x9a\x62\x52\x04\xde\xe1\xd1\x44\x5e\x4b\x4b\xc1\x10\xf1\xbe\x06\xc8\xf7\x07\xd2\x92\xc8\x92\xb3\xb6\xa5\x79\x6e\x93\x6d\xfa\xd1\x68\x6e\xec\x25\x10\xcf\x4f\x8e\xd4\xfb\x3c\x56\x04\xbc\x6f\xbc\x65", 128u),
+ 96u, // fixed hash length
+ UINT32_C(1477867120), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x43\xd7\x5e\xff\x9b\x29\x34\x97\x50\x92\xb3\xcd\x38\xee\x3c\xdf\xc7\x7a\x76\x4e\x72\xaf\xb2\xdd\x37\x1c\x8e\x1c\xec\x08\x15\x72\x4f\xac\x2d\x67\x19\xc1\xc5\x6f\x52\x7d\x79\xe9\xa8\x3d\xcb\x3b\xdb\x4d\x29\x81\x89\xa8\x1f\xcb\xbf\xc6\x2f\x5a\xab\x9e\x05\xc6\xec\x42\x24\xf0\x9e\x2c\xb4\x7e\xc1\x85\x10\x3c\xc0\xd5\x99\x9a\x62\x52\x04\xde\xe1\xd1\x44\x5e\x4b\x4b\xc1\x10\xf1\xbe\x06\xc8\xf7\x07\xd2\x92\xc8\x92\xb3\xb6\xa5\x79\x6e\x93\x6d\xfa\xd1\x68\x6e\xec\x25\x10\xcf\x4f\x8e\xd4\xfb\x3c\x56\x04\xbc\x6f\xbc\x65"), // macro-calculated hash
+},
+{
+ std::string_view("\x43\xd7\x5e\xff\x9b\x29\x34\x97\x50\x92\xb3\xcd\x38\xee\x3c\xdf\xc7\x7a\x76\x4e\x72\xaf\xb2\xdd\x37\x1c\x8e\x1c\xec\x08\x15\x72\x4f\xac\x2d\x67\x19\xc1\xc5\x6f\x52\x7d\x79\xe9\xa8\x3d\xcb\x3b\xdb\x4d\x29\x81\x89\xa8\x1f\xcb\xbf\xc6\x2f\x5a\xab\x9e\x05\xc6\xec\x42\x24\xf0\x9e\x2c\xb4\x7e\xc1\x85\x10\x3c\xc0\xd5\x99\x9a\x62\x52\x04\xde\xe1\xd1\x44\x5e\x4b\x4b\xc1\x10\xf1\xbe\x06\xc8\xf7\x07\xd2\x92\xc8\x92\xb3\xb6\xa5\x79\x6e\x93\x6d\xfa\xd1\x68\x6e\xec\x25\x10\xcf\x4f\x8e\xd4\xfb\x3c\x56\x04\xbc\x6f\xbc\x65", 128u),
+ 128u, // fixed hash length
+ UINT32_C(3694995364), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x43\xd7\x5e\xff\x9b\x29\x34\x97\x50\x92\xb3\xcd\x38\xee\x3c\xdf\xc7\x7a\x76\x4e\x72\xaf\xb2\xdd\x37\x1c\x8e\x1c\xec\x08\x15\x72\x4f\xac\x2d\x67\x19\xc1\xc5\x6f\x52\x7d\x79\xe9\xa8\x3d\xcb\x3b\xdb\x4d\x29\x81\x89\xa8\x1f\xcb\xbf\xc6\x2f\x5a\xab\x9e\x05\xc6\xec\x42\x24\xf0\x9e\x2c\xb4\x7e\xc1\x85\x10\x3c\xc0\xd5\x99\x9a\x62\x52\x04\xde\xe1\xd1\x44\x5e\x4b\x4b\xc1\x10\xf1\xbe\x06\xc8\xf7\x07\xd2\x92\xc8\x92\xb3\xb6\xa5\x79\x6e\x93\x6d\xfa\xd1\x68\x6e\xec\x25\x10\xcf\x4f\x8e\xd4\xfb\x3c\x56\x04\xbc\x6f\xbc\x65"), // macro-calculated hash
+},
+{
+ std::string_view("\x23\x66\x8c\x45\x63\x79\xfb\x31\x74\x7c\xa5\xcc\x58\x09\xc6\x46\x13\x9e\xf1\x96\x66\xb5\x03\x3a\xaa\x03\x78\x6e\x93\xdb\x9e\x70\x74\x05\xaa\x30\xca\x00\xf3\xc3\xef\xd0\x2e\xb5\xc3\x6c\xbd\x7f\xbc\x41\xd2\x1a\x83\xec\x72\xb0\x8c\x35\x9e\xcf\x6f\x16\x90\x6b\xdb\xa5\x88\xc6\xdc\x05\x8c\xdf\x51\x99\xa7\xca\xa9\xe3\x59\x68\x20\xb2\xdd\x63\xab\xd9\x27\xec\xcb\x1a\x2c\xac\xed\x5d\x8c\x97\x04\xbe\x22\x76\xbb\x14\x3b\x5f\xff\xc3\x78\xe2\xed\x79\xea\xdf\xc4\x5a\x28\xf2\x7a\xcb\x20\x68\x7b\xdc\xbf\xf2\x77\x61\x56\x5a\xe1", 129u),
+ 80u, // fixed hash length
+ UINT32_C(6281856), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_80_HASH("\x23\x66\x8c\x45\x63\x79\xfb\x31\x74\x7c\xa5\xcc\x58\x09\xc6\x46\x13\x9e\xf1\x96\x66\xb5\x03\x3a\xaa\x03\x78\x6e\x93\xdb\x9e\x70\x74\x05\xaa\x30\xca\x00\xf3\xc3\xef\xd0\x2e\xb5\xc3\x6c\xbd\x7f\xbc\x41\xd2\x1a\x83\xec\x72\xb0\x8c\x35\x9e\xcf\x6f\x16\x90\x6b\xdb\xa5\x88\xc6\xdc\x05\x8c\xdf\x51\x99\xa7\xca\xa9\xe3\x59\x68\x20\xb2\xdd\x63\xab\xd9\x27\xec\xcb\x1a\x2c\xac\xed\x5d\x8c\x97\x04\xbe\x22\x76\xbb\x14\x3b\x5f\xff\xc3\x78\xe2\xed\x79\xea\xdf\xc4\x5a\x28\xf2\x7a\xcb\x20\x68\x7b\xdc\xbf\xf2\x77\x61\x56\x5a\xe1"), // macro-calculated hash
+},
+{
+ std::string_view("\x23\x66\x8c\x45\x63\x79\xfb\x31\x74\x7c\xa5\xcc\x58\x09\xc6\x46\x13\x9e\xf1\x96\x66\xb5\x03\x3a\xaa\x03\x78\x6e\x93\xdb\x9e\x70\x74\x05\xaa\x30\xca\x00\xf3\xc3\xef\xd0\x2e\xb5\xc3\x6c\xbd\x7f\xbc\x41\xd2\x1a\x83\xec\x72\xb0\x8c\x35\x9e\xcf\x6f\x16\x90\x6b\xdb\xa5\x88\xc6\xdc\x05\x8c\xdf\x51\x99\xa7\xca\xa9\xe3\x59\x68\x20\xb2\xdd\x63\xab\xd9\x27\xec\xcb\x1a\x2c\xac\xed\x5d\x8c\x97\x04\xbe\x22\x76\xbb\x14\x3b\x5f\xff\xc3\x78\xe2\xed\x79\xea\xdf\xc4\x5a\x28\xf2\x7a\xcb\x20\x68\x7b\xdc\xbf\xf2\x77\x61\x56\x5a\xe1", 129u),
+ 96u, // fixed hash length
+ UINT32_C(598421397), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_96_HASH("\x23\x66\x8c\x45\x63\x79\xfb\x31\x74\x7c\xa5\xcc\x58\x09\xc6\x46\x13\x9e\xf1\x96\x66\xb5\x03\x3a\xaa\x03\x78\x6e\x93\xdb\x9e\x70\x74\x05\xaa\x30\xca\x00\xf3\xc3\xef\xd0\x2e\xb5\xc3\x6c\xbd\x7f\xbc\x41\xd2\x1a\x83\xec\x72\xb0\x8c\x35\x9e\xcf\x6f\x16\x90\x6b\xdb\xa5\x88\xc6\xdc\x05\x8c\xdf\x51\x99\xa7\xca\xa9\xe3\x59\x68\x20\xb2\xdd\x63\xab\xd9\x27\xec\xcb\x1a\x2c\xac\xed\x5d\x8c\x97\x04\xbe\x22\x76\xbb\x14\x3b\x5f\xff\xc3\x78\xe2\xed\x79\xea\xdf\xc4\x5a\x28\xf2\x7a\xcb\x20\x68\x7b\xdc\xbf\xf2\x77\x61\x56\x5a\xe1"), // macro-calculated hash
+},
+{
+ std::string_view("\x23\x66\x8c\x45\x63\x79\xfb\x31\x74\x7c\xa5\xcc\x58\x09\xc6\x46\x13\x9e\xf1\x96\x66\xb5\x03\x3a\xaa\x03\x78\x6e\x93\xdb\x9e\x70\x74\x05\xaa\x30\xca\x00\xf3\xc3\xef\xd0\x2e\xb5\xc3\x6c\xbd\x7f\xbc\x41\xd2\x1a\x83\xec\x72\xb0\x8c\x35\x9e\xcf\x6f\x16\x90\x6b\xdb\xa5\x88\xc6\xdc\x05\x8c\xdf\x51\x99\xa7\xca\xa9\xe3\x59\x68\x20\xb2\xdd\x63\xab\xd9\x27\xec\xcb\x1a\x2c\xac\xed\x5d\x8c\x97\x04\xbe\x22\x76\xbb\x14\x3b\x5f\xff\xc3\x78\xe2\xed\x79\xea\xdf\xc4\x5a\x28\xf2\x7a\xcb\x20\x68\x7b\xdc\xbf\xf2\x77\x61\x56\x5a\xe1", 129u),
+ 128u, // fixed hash length
+ UINT32_C(1313299978), // Python-calculated hash
+ PW_TOKENIZER_65599_FIXED_LENGTH_128_HASH("\x23\x66\x8c\x45\x63\x79\xfb\x31\x74\x7c\xa5\xcc\x58\x09\xc6\x46\x13\x9e\xf1\x96\x66\xb5\x03\x3a\xaa\x03\x78\x6e\x93\xdb\x9e\x70\x74\x05\xaa\x30\xca\x00\xf3\xc3\xef\xd0\x2e\xb5\xc3\x6c\xbd\x7f\xbc\x41\xd2\x1a\x83\xec\x72\xb0\x8c\x35\x9e\xcf\x6f\x16\x90\x6b\xdb\xa5\x88\xc6\xdc\x05\x8c\xdf\x51\x99\xa7\xca\xa9\xe3\x59\x68\x20\xb2\xdd\x63\xab\xd9\x27\xec\xcb\x1a\x2c\xac\xed\x5d\x8c\x97\x04\xbe\x22\x76\xbb\x14\x3b\x5f\xff\xc3\x78\xe2\xed\x79\xea\xdf\xc4\x5a\x28\xf2\x7a\xcb\x20\x68\x7b\xdc\xbf\xf2\x77\x61\x56\x5a\xe1"), // macro-calculated hash
+},
+
+}; // kHashTests
+
+// clang-format on
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/pw_tokenizer_private/tokenize_test.h b/pw_tokenizer/pw_tokenizer_private/tokenize_test.h
new file mode 100644
index 0000000..7ed6f15
--- /dev/null
+++ b/pw_tokenizer/pw_tokenizer_private/tokenize_test.h
@@ -0,0 +1,46 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// Functions to test that the tokenization macro works correctly in C code.
+#pragma once
+
+#include <stddef.h>
+#include <stdint.h>
+
+#include "pw_preprocessor/util.h"
+
+PW_EXTERN_C_START
+
+#define TEST_FORMAT_STRING_SHORT_FLOAT "Hello %s! %hd %e"
+
+void pw_TokenizeToBufferTest_StringShortFloat(void* buffer,
+ size_t* buffer_size);
+
+#define TEST_FORMAT_SEQUENTIAL_ZIG_ZAG "%u%d%02x%X%hu%hhd%d%ld%lu%lld%llu%c%c%c"
+
+void pw_TokenizeToBufferTest_SequentialZigZag(void* buffer,
+ size_t* buffer_size);
+
+void pw_TokenizeToCallbackTest_SequentialZigZag(
+ void (*callback)(const uint8_t* buffer, size_t size));
+
+void pw_TokenizeToGlobalHandlerTest_SequentialZigZag(void);
+
+void pw_TokenizeToGlobalHandlerWithPayloadTest_SequentialZigZag(void);
+
+#define TEST_FORMAT_REQUIRES_8 "Won't fit : %s%d"
+
+void pw_TokenizeToBufferTest_Requires8(void* buffer, size_t* buffer_size);
+
+PW_EXTERN_C_END
diff --git a/pw_tokenizer/pw_tokenizer_private/tokenized_string_decoding_test_data.h b/pw_tokenizer/pw_tokenizer_private/tokenized_string_decoding_test_data.h
new file mode 100644
index 0000000..03d2021
--- /dev/null
+++ b/pw_tokenizer/pw_tokenizer_private/tokenized_string_decoding_test_data.h
@@ -0,0 +1,417 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// AUTOGENERATED - DO NOT EDIT
+// This file contains test data generated by generate_decoding_test_data.cc.
+#pragma once
+
+#include <string_view>
+#include <tuple>
+
+namespace pw::test::tokenized_string_decoding {
+
+using namespace std::literals::string_view_literals;
+
+// clang-format off
+using TestCase = std::tuple<const char*, std::string_view, std::string_view>;
+
+inline constexpr TestCase kTestData[] = {
+
+// Simple strings
+TestCase("%s", "SFO", "\x03\x53\x46\x4f"sv),
+TestCase("%s", "KSJC", "\x04\x4b\x53\x4a\x43"sv),
+TestCase("%s", "", "\x00"sv),
+TestCase("%5s%s", " nofun", "\x02\x6e\x6f\x03\x66\x75\x6e"sv),
+TestCase("%5s%s", "abcdef", "\x06\x61\x62\x63\x64\x65\x66\x00"sv),
+TestCase("%5s%s", " abcdef", "\x00\x06\x61\x62\x63\x64\x65\x66"sv),
+TestCase("%s %-6s%s%s%s", "Intel 80586 toaster oven", "\x05\x49\x6e\x74\x65\x6c\x05\x38\x30\x35\x38\x36\x07\x74\x6f\x61\x73\x74\x65\x72\x01\x20\x04\x6f\x76\x65\x6e"sv),
+TestCase("%s %-6s%s%s%s", "Apple automatic pencil sharpener", "\x05\x41\x70\x70\x6c\x65\x09\x61\x75\x74\x6f\x6d\x61\x74\x69\x63\x07\x20\x70\x65\x6e\x63\x69\x6c\x01\x20\x09\x73\x68\x61\x72\x70\x65\x6e\x65\x72"sv),
+
+// Zero-length strings
+TestCase("%s-%s", "so-", "\x02\x73\x6f\x00"sv),
+TestCase("%s-%s", "-cool", "\x00\x04\x63\x6f\x6f\x6c"sv),
+TestCase("%s%s%3s%s", " ", "\x00\x00\x00\x00"sv),
+TestCase("(%5s)(%2s)(%7s)", "([...])( )( [...])", "\x80\x00\x80"sv),
+
+// Invalid strings
+TestCase("%s", "<[%s ERROR (hi)]>", "\x03\x68\x69"sv),
+TestCase("%30s", "<[%30s ERROR (hi)]>", "\x03\x68\x69"sv),
+TestCase("%30s", "<[%30s ERROR (hi)]>", "\x83\x68\x69"sv),
+TestCase("%s", "<[%s ERROR (yo!)]>", "\x85\x79\x6f\x21"sv),
+TestCase("%s", "<[%s ERROR]>", "\x01"sv),
+TestCase("%30s", "<[%30s ERROR]>", "\x81"sv),
+
+// Continue after truncated string
+TestCase("%s %d %s", "go[...] 2 lunch", "\x82\x67\x6f\x04\x05\x6c\x75\x6e\x63\x68"sv),
+TestCase("%6s%s%s", " [...]hello[...]there", "\x80\x85\x68\x65\x6c\x6c\x6f\x05\x74\x68\x65\x72\x65"sv),
+
+// Floating point
+TestCase("%1.1f", "0.0", "\x00\x00\x00\x00"sv),
+TestCase("%0.5f", "3.14159", "\xdb\x0f\x49\x40"sv),
+
+// Character
+TestCase("%c", " ", "\x40"sv),
+TestCase("%c", "$", "\x48"sv),
+TestCase("%c", "$", "\x48"sv),
+TestCase("100%c!", "100%!", "\x4a"sv),
+
+// Atypical argument types
+TestCase("%ju", "99", "\xc6\x01"sv),
+TestCase("%jd", "99", "\xc6\x01"sv),
+TestCase("%zu", "8", "\x10"sv),
+TestCase("%zd", "123", "\xf6\x01"sv),
+TestCase("%td", "99", "\xc6\x01"sv),
+
+// Percent character
+TestCase("%%", "%", ""sv),
+TestCase("%%%%%%%%", "%%%%", "\x61\x62\x63"sv),
+TestCase("whoa%%%%wow%%%%!%%", "whoa%%wow%%!%", ""sv),
+TestCase("This is %d%% effective", "This is 1% effective", "\x02"sv),
+TestCase("%% is 100%sa%%sign%%%s", "% is 100%a%sign%OK?", "\x01\x25\x03\x4f\x4b\x3f"sv),
+
+// Percent character prints after errors
+TestCase("%s%%", "-10[...]%", "\x83\x2d\x31\x30\x00"sv),
+TestCase("%d%% is a good %%", "<[%d MISSING]>% is a good %", ""sv),
+
+// Various format strings
+TestCase("!", "!", ""sv),
+TestCase("%s", "%s", "\x02\x25\x73"sv),
+TestCase("%s", "hello", "\x05\x68\x65\x6c\x6c\x6f"sv),
+TestCase("%s%s", "Helloold", "\x05\x48\x65\x6c\x6c\x6f\x03\x6f\x6c\x64"sv),
+TestCase("%s to the%c%s", "hello to the whirled", "\x05\x68\x65\x6c\x6c\x6f\x40\x07\x77\x68\x69\x72\x6c\x65\x64"sv),
+TestCase("hello %s %d %d %d", "hello rolled 1 2 3", "\x06\x72\x6f\x6c\x6c\x65\x64\x02\x04\x06"sv),
+TestCase("", "", ""sv),
+TestCase("This has no specifiers", "This has no specifiers", ""sv),
+TestCase("%s_or_%3s", "hello_or_ hi", "\x05\x68\x65\x6c\x6c\x6f\x02\x68\x69"sv),
+TestCase("%s_or_%3d", "hello_or_-64", "\x05\x68\x65\x6c\x6c\x6f\x7f"sv),
+TestCase("%s or hi%c pi=%1.2e", "hello or hi! pi=3.14e+00", "\x05\x68\x65\x6c\x6c\x6f\x42\xdb\x0f\x49\x40"sv),
+TestCase("Why, %s there. My favorite number is %.2f%c", "Why, hello there. My favorite number is 3.14!", "\x05\x68\x65\x6c\x6c\x6f\xdb\x0f\x49\x40\x42"sv),
+
+// Various errors
+TestCase("%d", "<[%d MISSING]>", ""sv),
+TestCase("ABC%d123%dabc%dABC", "ABC<[%d MISSING]>123<[%d SKIPPED]>abc<[%d SKIPPED]>ABC", ""sv),
+TestCase("%sXY%+ldxy%a", "Yo![...]XY<[%+ld ERROR]>xy<[%a SKIPPED]>", "\x83\x59\x6f\x21\x80"sv),
+TestCase("%d", "<[%d MISSING]>", ""sv),
+TestCase("%sXY%+ldxy%a", "Yo![...]XY<[%+ld ERROR]>xy<[%a SKIPPED]>", "\x83\x59\x6f\x21\x80"sv),
+TestCase("%s%lld%9u", "$[...]<[%lld ERROR]><[%9u SKIPPED]>", "\x81\x24\x80\x80"sv),
+
+// Alternate form (#)
+TestCase("Hex: %#x", "Hex: 0xbeef", "\xde\xfb\x05"sv),
+TestCase("Hex: %#08X", "Hex: 0X00FEED", "\xda\xfb\x07"sv),
+
+// Random integers
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.210e-013E-01000.3210430.3210430.3210430.321043", "\xcb\x5f\xa4\x3e\xcb\x5f\xa4\x3e\x00\xcb\x5f\xa4\x3e\xcb\x5f\xa4\x3e\xcb\x5f\xa4\x3e\xcb\x5f\xa4\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.914e-012E-01010.191430.191430.1914300.191430", "\x4c\x06\x44\x3e\x4c\x06\x44\x3e\x02\x4c\x06\x44\x3e\x4c\x06\x44\x3e\x4c\x06\x44\x3e\x4c\x06\x44\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.087e-011E-01020.108680.108680.1086800.108680", "\xb5\x93\xde\x3d\xb5\x93\xde\x3d\x04\xb5\x93\xde\x3d\xb5\x93\xde\x3d\xb5\x93\xde\x3d\xb5\x93\xde\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.821e-012E-01030.1821430.1821430.1821430.182143", "\xd3\x83\x3a\x3e\xd3\x83\x3a\x3e\x06\xd3\x83\x3a\x3e\xd3\x83\x3a\x3e\xd3\x83\x3a\x3e\xd3\x83\x3a\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +4.159e-014E-01040.4159270.4159270.4159270.415927", "\x56\xf4\xd4\x3e\x56\xf4\xd4\x3e\x08\x56\xf4\xd4\x3e\x56\xf4\xd4\x3e\x56\xf4\xd4\x3e\x56\xf4\xd4\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.019e-019E-01050.9018610.9018610.9018610.901861", "\x64\xe0\x66\x3f\x64\xe0\x66\x3f\x0a\x64\xe0\x66\x3f\x64\xe0\x66\x3f\x64\xe0\x66\x3f\x64\xe0\x66\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.429e-013E-01060.3429150.3429150.3429150.342915", "\x8c\x92\xaf\x3e\x8c\x92\xaf\x3e\x0c\x8c\x92\xaf\x3e\x8c\x92\xaf\x3e\x8c\x92\xaf\x3e\x8c\x92\xaf\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +7.011e-017E-01070.7011480.7011480.7011480.701148", "\x68\x7e\x33\x3f\x68\x7e\x33\x3f\x0e\x68\x7e\x33\x3f\x68\x7e\x33\x3f\x68\x7e\x33\x3f\x68\x7e\x33\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.988e-017E-01080.6988460.6988460.6988460.698846", "\x96\xe7\x32\x3f\x96\xe7\x32\x3f\x10\x96\xe7\x32\x3f\x96\xe7\x32\x3f\x96\xe7\x32\x3f\x96\xe7\x32\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.383e-018E-01090.8383340.8383340.8383340.838334", "\x0e\x9d\x56\x3f\x0e\x9d\x56\x3f\x12\x0e\x9d\x56\x3f\x0e\x9d\x56\x3f\x0e\x9d\x56\x3f\x0e\x9d\x56\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.469e-012E-01100.2468540.2468540.2468540.246854", "\x6b\xc7\x7c\x3e\x6b\xc7\x7c\x3e\x14\x6b\xc7\x7c\x3e\x6b\xc7\x7c\x3e\x6b\xc7\x7c\x3e\x6b\xc7\x7c\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.634e-011E+00110.9634290.9634290.9634290.963429", "\x4a\xa3\x76\x3f\x4a\xa3\x76\x3f\x16\x4a\xa3\x76\x3f\x4a\xa3\x76\x3f\x4a\xa3\x76\x3f\x4a\xa3\x76\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.454e-013E-01120.3454440.3454440.3454440.345444", "\xf9\xdd\xb0\x3e\xf9\xdd\xb0\x3e\x18\xf9\xdd\xb0\x3e\xf9\xdd\xb0\x3e\xf9\xdd\xb0\x3e\xf9\xdd\xb0\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.344e-019E-01130.9344260.9344260.9344260.934426", "\x83\x36\x6f\x3f\x83\x36\x6f\x3f\x1a\x83\x36\x6f\x3f\x83\x36\x6f\x3f\x83\x36\x6f\x3f\x83\x36\x6f\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.880e-014E-01140.3879820.3879820.3879820.387982", "\x88\xa5\xc6\x3e\x88\xa5\xc6\x3e\x1c\x88\xa5\xc6\x3e\x88\xa5\xc6\x3e\x88\xa5\xc6\x3e\x88\xa5\xc6\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.852e-012E-01150.1851670.1851670.1851670.185167", "\x7c\x9c\x3d\x3e\x7c\x9c\x3d\x3e\x1e\x7c\x9c\x3d\x3e\x7c\x9c\x3d\x3e\x7c\x9c\x3d\x3e\x7c\x9c\x3d\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.251e-015E-01160.5250540.5250540.5250540.525054", "\xe9\x69\x06\x3f\xe9\x69\x06\x3f\x20\xe9\x69\x06\x3f\xe9\x69\x06\x3f\xe9\x69\x06\x3f\xe9\x69\x06\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +7.463e-027E-02170.07463110.07463110.0746310.074631", "\x2a\xd8\x98\x3d\x2a\xd8\x98\x3d\x22\x2a\xd8\x98\x3d\x2a\xd8\x98\x3d\x2a\xd8\x98\x3d\x2a\xd8\x98\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.482e-018E-01180.8481530.8481530.8481530.848153", "\x88\x20\x59\x3f\x88\x20\x59\x3f\x24\x88\x20\x59\x3f\x88\x20\x59\x3f\x88\x20\x59\x3f\x88\x20\x59\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.654e-012E-01190.1654280.1654280.1654280.165428", "\xda\x65\x29\x3e\xda\x65\x29\x3e\x26\xda\x65\x29\x3e\xda\x65\x29\x3e\xda\x65\x29\x3e\xda\x65\x29\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.280e-016E-01200.627980.627980.6279800.627980", "\x47\xc3\x20\x3f\x47\xc3\x20\x3f\x28\x47\xc3\x20\x3f\x47\xc3\x20\x3f\x47\xc3\x20\x3f\x47\xc3\x20\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.176e-016E-01210.6175810.6175810.6175810.617581", "\xd2\x19\x1e\x3f\xd2\x19\x1e\x3f\x2a\xd2\x19\x1e\x3f\xd2\x19\x1e\x3f\xd2\x19\x1e\x3f\xd2\x19\x1e\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.034e-015E-01220.5033950.5033950.5033950.503395", "\x85\xde\x00\x3f\x85\xde\x00\x3f\x2c\x85\xde\x00\x3f\x85\xde\x00\x3f\x85\xde\x00\x3f\x85\xde\x00\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.409e-013E-01230.340910.340910.3409100.340910", "\xcb\x8b\xae\x3e\xcb\x8b\xae\x3e\x2e\xcb\x8b\xae\x3e\xcb\x8b\xae\x3e\xcb\x8b\xae\x3e\xcb\x8b\xae\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.797e-026E-02240.05796970.05796970.0579700.057970", "\x9d\x71\x6d\x3d\x9d\x71\x6d\x3d\x30\x9d\x71\x6d\x3d\x9d\x71\x6d\x3d\x9d\x71\x6d\x3d\x9d\x71\x6d\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.424e-019E-01250.9423620.9423620.9423620.942362", "\x9e\x3e\x71\x3f\x9e\x3e\x71\x3f\x32\x9e\x3e\x71\x3f\x9e\x3e\x71\x3f\x9e\x3e\x71\x3f\x9e\x3e\x71\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.129e-016E-01260.6128640.6128640.6128640.612864", "\xa9\xe4\x1c\x3f\xa9\xe4\x1c\x3f\x34\xa9\xe4\x1c\x3f\xa9\xe4\x1c\x3f\xa9\xe4\x1c\x3f\xa9\xe4\x1c\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.093e-013E-01270.3093140.3093140.3093140.309314", "\x5e\x5e\x9e\x3e\x5e\x5e\x9e\x3e\x36\x5e\x5e\x9e\x3e\x5e\x5e\x9e\x3e\x5e\x5e\x9e\x3e\x5e\x5e\x9e\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +7.864e-018E-01280.7863820.7863820.7863820.786382", "\x4e\x50\x49\x3f\x4e\x50\x49\x3f\x38\x4e\x50\x49\x3f\x4e\x50\x49\x3f\x4e\x50\x49\x3f\x4e\x50\x49\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.105e-015E-01290.5104890.5104890.5104890.510489", "\x65\xaf\x02\x3f\x65\xaf\x02\x3f\x3a\x65\xaf\x02\x3f\x65\xaf\x02\x3f\x65\xaf\x02\x3f\x65\xaf\x02\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.358e-011E-01300.1357620.1357620.1357620.135762", "\x16\x05\x0b\x3e\x16\x05\x0b\x3e\x3c\x16\x05\x0b\x3e\x16\x05\x0b\x3e\x16\x05\x0b\x3e\x16\x05\x0b\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.879e-011E+00310.987870.987870.9878700.987870", "\x14\xe5\x7c\x3f\x14\xe5\x7c\x3f\x3e\x14\xe5\x7c\x3f\x14\xe5\x7c\x3f\x14\xe5\x7c\x3f\x14\xe5\x7c\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.994e-012E-01320.1994450.1994450.1994450.199445", "\x48\x3b\x4c\x3e\x48\x3b\x4c\x3e\x40\x48\x3b\x4c\x3e\x48\x3b\x4c\x3e\x48\x3b\x4c\x3e\x48\x3b\x4c\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.042e-025E-02330.0504220.0504220.0504220.050422", "\x47\x87\x4e\x3d\x47\x87\x4e\x3d\x42\x47\x87\x4e\x3d\x47\x87\x4e\x3d\x47\x87\x4e\x3d\x47\x87\x4e\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +7.875e-028E-02340.07875130.07875130.0787510.078751", "\x59\x48\xa1\x3d\x59\x48\xa1\x3d\x44\x59\x48\xa1\x3d\x59\x48\xa1\x3d\x59\x48\xa1\x3d\x59\x48\xa1\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.121e-019E-01350.9120510.9120510.9120510.912051", "\x2b\x7c\x69\x3f\x2b\x7c\x69\x3f\x46\x2b\x7c\x69\x3f\x2b\x7c\x69\x3f\x2b\x7c\x69\x3f\x2b\x7c\x69\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.001e-011E-01360.1000970.1000970.1000970.100097", "\xaf\xff\xcc\x3d\xaf\xff\xcc\x3d\x48\xaf\xff\xcc\x3d\xaf\xff\xcc\x3d\xaf\xff\xcc\x3d\xaf\xff\xcc\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.278e-015E-01370.5277510.5277510.5277510.527751", "\xaa\x1a\x07\x3f\xaa\x1a\x07\x3f\x4a\xaa\x1a\x07\x3f\xaa\x1a\x07\x3f\xaa\x1a\x07\x3f\xaa\x1a\x07\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +7.040e-017E-01380.7039560.7039560.7039560.703956", "\x76\x36\x34\x3f\x76\x36\x34\x3f\x4c\x76\x36\x34\x3f\x76\x36\x34\x3f\x76\x36\x34\x3f\x76\x36\x34\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.794e-019E-01390.8793580.8793580.8793580.879358", "\x96\x1d\x61\x3f\x96\x1d\x61\x3f\x4e\x96\x1d\x61\x3f\x96\x1d\x61\x3f\x96\x1d\x61\x3f\x96\x1d\x61\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.455e-012E-01400.2454720.2454720.2454720.245472", "\x1c\x5d\x7b\x3e\x1c\x5d\x7b\x3e\x50\x1c\x5d\x7b\x3e\x1c\x5d\x7b\x3e\x1c\x5d\x7b\x3e\x1c\x5d\x7b\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.607e-017E-01410.6606620.6606620.6606620.660662", "\x1f\x21\x29\x3f\x1f\x21\x29\x3f\x52\x1f\x21\x29\x3f\x1f\x21\x29\x3f\x1f\x21\x29\x3f\x1f\x21\x29\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +4.550e-015E-01420.4550340.4550340.4550340.455034", "\x46\xfa\xe8\x3e\x46\xfa\xe8\x3e\x54\x46\xfa\xe8\x3e\x46\xfa\xe8\x3e\x46\xfa\xe8\x3e\x46\xfa\xe8\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.344e-015E-01430.5343630.5343630.5343630.534363", "\x0b\xcc\x08\x3f\x0b\xcc\x08\x3f\x56\x0b\xcc\x08\x3f\x0b\xcc\x08\x3f\x0b\xcc\x08\x3f\x0b\xcc\x08\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.146e-012E-01440.2145620.2145620.2145620.214562", "\x09\xb6\x5b\x3e\x09\xb6\x5b\x3e\x58\x09\xb6\x5b\x3e\x09\xb6\x5b\x3e\x09\xb6\x5b\x3e\x09\xb6\x5b\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.157e-015E-01450.5156980.5156980.5156980.515698", "\xc3\x04\x04\x3f\xc3\x04\x04\x3f\x5a\xc3\x04\x04\x3f\xc3\x04\x04\x3f\xc3\x04\x04\x3f\xc3\x04\x04\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +7.148e-017E-01460.7147680.7147680.7147680.714768", "\x0b\xfb\x36\x3f\x0b\xfb\x36\x3f\x5c\x0b\xfb\x36\x3f\x0b\xfb\x36\x3f\x0b\xfb\x36\x3f\x0b\xfb\x36\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.730e-012E-01470.172960.172960.1729600.172960", "\x62\x1c\x31\x3e\x62\x1c\x31\x3e\x5e\x62\x1c\x31\x3e\x62\x1c\x31\x3e\x62\x1c\x31\x3e\x62\x1c\x31\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.971e-017E-01480.6970990.6970990.6970990.697099", "\x1b\x75\x32\x3f\x1b\x75\x32\x3f\x60\x1b\x75\x32\x3f\x1b\x75\x32\x3f\x1b\x75\x32\x3f\x1b\x75\x32\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.768e-013E-01490.2767580.2767580.2767580.276758", "\x31\xb3\x8d\x3e\x31\xb3\x8d\x3e\x62\x31\xb3\x8d\x3e\x31\xb3\x8d\x3e\x31\xb3\x8d\x3e\x31\xb3\x8d\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.734e-012E-01500.1734190.1734190.1734190.173419", "\xa2\x94\x31\x3e\xa2\x94\x31\x3e\x64\xa2\x94\x31\x3e\xa2\x94\x31\x3e\xa2\x94\x31\x3e\xa2\x94\x31\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.692e-011E+00510.9692260.9692260.9692260.969226", "\x32\x1f\x78\x3f\x32\x1f\x78\x3f\x66\x32\x1f\x78\x3f\x32\x1f\x78\x3f\x32\x1f\x78\x3f\x32\x1f\x78\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.781e-019E-01520.8781310.8781310.8781310.878131", "\x35\xcd\x60\x3f\x35\xcd\x60\x3f\x68\x35\xcd\x60\x3f\x35\xcd\x60\x3f\x35\xcd\x60\x3f\x35\xcd\x60\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.279e-013E-01530.3279260.3279260.3279260.327926", "\xf6\xe5\xa7\x3e\xf6\xe5\xa7\x3e\x6a\xf6\xe5\xa7\x3e\xf6\xe5\xa7\x3e\xf6\xe5\xa7\x3e\xf6\xe5\xa7\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +7.008e-017E-01540.7007780.7007780.7007780.700778", "\x31\x66\x33\x3f\x31\x66\x33\x3f\x6c\x31\x66\x33\x3f\x31\x66\x33\x3f\x31\x66\x33\x3f\x31\x66\x33\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.957e-011E+00550.9956590.9956590.9956590.995659", "\x84\xe3\x7e\x3f\x84\xe3\x7e\x3f\x6e\x84\xe3\x7e\x3f\x84\xe3\x7e\x3f\x84\xe3\x7e\x3f\x84\xe3\x7e\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.110e-012E-01560.2110.2110.2110000.211000", "\x6a\x10\x58\x3e\x6a\x10\x58\x3e\x70\x6a\x10\x58\x3e\x6a\x10\x58\x3e\x6a\x10\x58\x3e\x6a\x10\x58\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.794e-016E-01570.579380.579380.5793800.579380", "\x3a\x52\x14\x3f\x3a\x52\x14\x3f\x72\x3a\x52\x14\x3f\x3a\x52\x14\x3f\x3a\x52\x14\x3f\x3a\x52\x14\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.704e-014E-01580.3703670.3703670.3703670.370367", "\xb4\xa0\xbd\x3e\xb4\xa0\xbd\x3e\x74\xb4\xa0\xbd\x3e\xb4\xa0\xbd\x3e\xb4\xa0\xbd\x3e\xb4\xa0\xbd\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +1.978e-012E-01590.1977640.1977640.1977640.197764", "\x95\x82\x4a\x3e\x95\x82\x4a\x3e\x76\x95\x82\x4a\x3e\x95\x82\x4a\x3e\x95\x82\x4a\x3e\x95\x82\x4a\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.440e-018E-01600.8439990.8439990.8439990.843999", "\x4d\x10\x58\x3f\x4d\x10\x58\x3f\x78\x4d\x10\x58\x3f\x4d\x10\x58\x3f\x4d\x10\x58\x3f\x4d\x10\x58\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.444e-019E-01610.9443610.9443610.9443610.944361", "\xa9\xc1\x71\x3f\xa9\xc1\x71\x3f\x7a\xa9\xc1\x71\x3f\xa9\xc1\x71\x3f\xa9\xc1\x71\x3f\xa9\xc1\x71\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +4.393e-024E-02620.04392670.04392670.0439270.043927", "\x72\xec\x33\x3d\x72\xec\x33\x3d\x7c\x72\xec\x33\x3d\x72\xec\x33\x3d\x72\xec\x33\x3d\x72\xec\x33\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.167e-018E-01630.8167210.8167210.8167210.816721", "\xa2\x14\x51\x3f\xa2\x14\x51\x3f\x7e\xa2\x14\x51\x3f\xa2\x14\x51\x3f\xa2\x14\x51\x3f\xa2\x14\x51\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.951e-019E-01640.8951460.8951460.8951460.895146", "\x43\x28\x65\x3f\x43\x28\x65\x3f\x80\x01\x43\x28\x65\x3f\x43\x28\x65\x3f\x43\x28\x65\x3f\x43\x28\x65\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.260e-025E-02650.05259920.05259920.0525990.052599", "\x38\x72\x57\x3d\x38\x72\x57\x3d\x82\x01\x38\x72\x57\x3d\x38\x72\x57\x3d\x38\x72\x57\x3d\x38\x72\x57\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.882e-019E-01660.8882180.8882180.8882180.888218", "\x3f\x62\x63\x3f\x3f\x62\x63\x3f\x84\x01\x3f\x62\x63\x3f\x3f\x62\x63\x3f\x3f\x62\x63\x3f\x3f\x62\x63\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +4.788e-015E-01670.4787930.4787930.4787930.478793", "\x6b\x24\xf5\x3e\x6b\x24\xf5\x3e\x86\x01\x6b\x24\xf5\x3e\x6b\x24\xf5\x3e\x6b\x24\xf5\x3e\x6b\x24\xf5\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.328e-026E-02680.06327770.06327770.0632780.063278", "\xbb\x97\x81\x3d\xbb\x97\x81\x3d\x88\x01\xbb\x97\x81\x3d\xbb\x97\x81\x3d\xbb\x97\x81\x3d\xbb\x97\x81\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.968e-019E-01690.896790.896790.8967900.896790", "\x07\x94\x65\x3f\x07\x94\x65\x3f\x8a\x01\x07\x94\x65\x3f\x07\x94\x65\x3f\x07\x94\x65\x3f\x07\x94\x65\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.153e-013E-01700.3153250.3153250.3153250.315325", "\x4c\x72\xa1\x3e\x4c\x72\xa1\x3e\x8c\x01\x4c\x72\xa1\x3e\x4c\x72\xa1\x3e\x4c\x72\xa1\x3e\x4c\x72\xa1\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.968e-019E-01710.8968210.8968210.8968210.896821", "\x0b\x96\x65\x3f\x0b\x96\x65\x3f\x8e\x01\x0b\x96\x65\x3f\x0b\x96\x65\x3f\x0b\x96\x65\x3f\x0b\x96\x65\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.917e-019E-01720.8916720.8916720.8916720.891672", "\x98\x44\x64\x3f\x98\x44\x64\x3f\x90\x01\x98\x44\x64\x3f\x98\x44\x64\x3f\x98\x44\x64\x3f\x98\x44\x64\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.630e-013E-01730.2629920.2629920.2629920.262992", "\xe0\xa6\x86\x3e\xe0\xa6\x86\x3e\x92\x01\xe0\xa6\x86\x3e\xe0\xa6\x86\x3e\xe0\xa6\x86\x3e\xe0\xa6\x86\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +7.461e-017E-01740.7460720.7460720.7460720.746072", "\x95\xfe\x3e\x3f\x95\xfe\x3e\x3f\x94\x01\x95\xfe\x3e\x3f\x95\xfe\x3e\x3f\x95\xfe\x3e\x3f\x95\xfe\x3e\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.495e-018E-01750.8495190.8495190.8495190.849519", "\x0f\x7a\x59\x3f\x0f\x7a\x59\x3f\x96\x01\x0f\x7a\x59\x3f\x0f\x7a\x59\x3f\x0f\x7a\x59\x3f\x0f\x7a\x59\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +4.244e-014E-01760.4244340.4244340.4244340.424434", "\x63\x4f\xd9\x3e\x63\x4f\xd9\x3e\x98\x01\x63\x4f\xd9\x3e\x63\x4f\xd9\x3e\x63\x4f\xd9\x3e\x63\x4f\xd9\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.790e-011E+00770.9789560.9789560.9789560.978956", "\xe2\x9c\x7a\x3f\xe2\x9c\x7a\x3f\x9a\x01\xe2\x9c\x7a\x3f\xe2\x9c\x7a\x3f\xe2\x9c\x7a\x3f\xe2\x9c\x7a\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.633e-016E-01780.5632670.5632670.5632670.563267", "\x48\x32\x10\x3f\x48\x32\x10\x3f\x9c\x01\x48\x32\x10\x3f\x48\x32\x10\x3f\x48\x32\x10\x3f\x48\x32\x10\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.106e-012E-01790.2106230.2106230.2106230.210623", "\x7d\xad\x57\x3e\x7d\xad\x57\x3e\x9e\x01\x7d\xad\x57\x3e\x7d\xad\x57\x3e\x7d\xad\x57\x3e\x7d\xad\x57\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.172e-013E-01800.3171850.3171850.3171850.317185", "\x22\x66\xa2\x3e\x22\x66\xa2\x3e\xa0\x01\x22\x66\xa2\x3e\x22\x66\xa2\x3e\x22\x66\xa2\x3e\x22\x66\xa2\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.624e-017E-01810.6623550.6623550.6623550.662355", "\x18\x90\x29\x3f\x18\x90\x29\x3f\xa2\x01\x18\x90\x29\x3f\x18\x90\x29\x3f\x18\x90\x29\x3f\x18\x90\x29\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.895e-014E-01820.3895240.3895240.3895240.389524", "\xba\x6f\xc7\x3e\xba\x6f\xc7\x3e\xa4\x01\xba\x6f\xc7\x3e\xba\x6f\xc7\x3e\xba\x6f\xc7\x3e\xba\x6f\xc7\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +9.884e-011E+00830.9883550.9883550.9883550.988355", "\xcd\x04\x7d\x3f\xcd\x04\x7d\x3f\xa6\x01\xcd\x04\x7d\x3f\xcd\x04\x7d\x3f\xcd\x04\x7d\x3f\xcd\x04\x7d\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.176e-012E-01840.2175680.2175680.2175680.217568", "\x0b\xca\x5e\x3e\x0b\xca\x5e\x3e\xa8\x01\x0b\xca\x5e\x3e\x0b\xca\x5e\x3e\x0b\xca\x5e\x3e\x0b\xca\x5e\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +4.721e-015E-01850.4721430.4721430.4721430.472143", "\xbb\xbc\xf1\x3e\xbb\xbc\xf1\x3e\xaa\x01\xbb\xbc\xf1\x3e\xbb\xbc\xf1\x3e\xbb\xbc\xf1\x3e\xbb\xbc\xf1\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.868e-019E-01860.8868160.8868160.8868160.886816", "\x62\x06\x63\x3f\x62\x06\x63\x3f\xac\x01\x62\x06\x63\x3f\x62\x06\x63\x3f\x62\x06\x63\x3f\x62\x06\x63\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.723e-019E-01870.8722940.8722940.8722940.872294", "\xa9\x4e\x5f\x3f\xa9\x4e\x5f\x3f\xae\x01\xa9\x4e\x5f\x3f\xa9\x4e\x5f\x3f\xa9\x4e\x5f\x3f\xa9\x4e\x5f\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.230e-025E-02880.05230090.05230090.0523010.052301", "\x6f\x39\x56\x3d\x6f\x39\x56\x3d\xb0\x01\x6f\x39\x56\x3d\x6f\x39\x56\x3d\x6f\x39\x56\x3d\x6f\x39\x56\x3d"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.598e-017E-01890.6598280.6598280.6598280.659828", "\x7f\xea\x28\x3f\x7f\xea\x28\x3f\xb2\x01\x7f\xea\x28\x3f\x7f\xea\x28\x3f\x7f\xea\x28\x3f\x7f\xea\x28\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.618e-016E-01900.5617660.5617660.5617660.561766", "\xe3\xcf\x0f\x3f\xe3\xcf\x0f\x3f\xb4\x01\xe3\xcf\x0f\x3f\xe3\xcf\x0f\x3f\xe3\xcf\x0f\x3f\xe3\xcf\x0f\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.281e-012E-01910.2280650.2280650.2280650.228065", "\xd6\x89\x69\x3e\xd6\x89\x69\x3e\xb6\x01\xd6\x89\x69\x3e\xd6\x89\x69\x3e\xd6\x89\x69\x3e\xd6\x89\x69\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.881e-013E-01920.2880970.2880970.2880970.288097", "\x69\x81\x93\x3e\x69\x81\x93\x3e\xb8\x01\x69\x81\x93\x3e\x69\x81\x93\x3e\x69\x81\x93\x3e\x69\x81\x93\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +7.210e-017E-01930.7209650.7209650.7209650.720965", "\x31\x91\x38\x3f\x31\x91\x38\x3f\xba\x01\x31\x91\x38\x3f\x31\x91\x38\x3f\x31\x91\x38\x3f\x31\x91\x38\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +2.152e-012E-01940.2152470.2152470.2152470.215247", "\xb3\x69\x5c\x3e\xb3\x69\x5c\x3e\xbc\x01\xb3\x69\x5c\x3e\xb3\x69\x5c\x3e\xb3\x69\x5c\x3e\xb3\x69\x5c\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +5.282e-015E-01950.5281530.5281530.5281530.528153", "\x0a\x35\x07\x3f\x0a\x35\x07\x3f\xbe\x01\x0a\x35\x07\x3f\x0a\x35\x07\x3f\x0a\x35\x07\x3f\x0a\x35\x07\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +6.896e-017E-01960.689570.689570.6895700.689570", "\xa6\x87\x30\x3f\xa6\x87\x30\x3f\xc0\x01\xa6\x87\x30\x3f\xa6\x87\x30\x3f\xa6\x87\x30\x3f\xa6\x87\x30\x3f"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.952e-014E-01970.3951650.3951650.3951650.395165", "\x0e\x53\xca\x3e\x0e\x53\xca\x3e\xc2\x01\x0e\x53\xca\x3e\x0e\x53\xca\x3e\x0e\x53\xca\x3e\x0e\x53\xca\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +3.759e-014E-01980.3759280.3759280.3759280.375928", "\x96\x79\xc0\x3e\x96\x79\xc0\x3e\xc4\x01\x96\x79\xc0\x3e\x96\x79\xc0\x3e\x96\x79\xc0\x3e\x96\x79\xc0\x3e"sv),
+TestCase("This is a number: %+08.3e%1.0E%02d%g%G%f%-3f", "This is a number: +8.436e-028E-02990.08436450.08436450.0843640.084364", "\x48\xc7\xac\x3d\x48\xc7\xac\x3d\xc6\x01\x48\xc7\xac\x3d\x48\xc7\xac\x3d\x48\xc7\xac\x3d\x48\xc7\xac\x3d"sv),
+TestCase("%s: %llu %d %c", "0: 646757550612212450 128431401 P", "\x01\x30\xc4\xab\xd7\xc0\x99\xa4\xdf\xf9\x11\xd2\xd4\xbd\x7a\xa0\x01"sv),
+TestCase("%s: %llu %d %c", "1: 5294527771240555016 1496480504 ", "\x01\x31\x90\x88\xdf\xa3\x9a\xbd\xf8\xf9\x92\x01\xf0\xeb\x93\x93\x0b\x12"sv),
+TestCase("%s: %llu %d %c", "2: 4061389961487213535 1555570883 >", "\x01\x32\xbe\x8f\xc3\xcc\xdb\xa9\xfa\xdc\x70\x86\x83\xc1\xcb\x0b\x7c"sv),
+TestCase("%s: %llu %d %c", "3: 7083829078452288754 1162540900 d", "\x01\x33\xe4\xf3\xd9\xeb\xfb\xc0\xe8\xce\xc4\x01\xc8\xdd\xd7\xd4\x08\xc8\x01"sv),
+TestCase("%s: %llu %d %c", "4: 4002626457632111277 1471392205 <", "\x01\x34\xda\xea\xef\xfb\xcd\xe3\x97\x8c\x6f\x9a\xa7\x9d\xfb\x0a\x78"sv),
+TestCase("%s: %llu %d %c", "5: 3705575452837642392 999262596 K", "\x01\x35\xb0\x82\xdc\xac\xb9\xcf\xec\xec\x66\x88\xa6\xfc\xb8\x07\x96\x01"sv),
+TestCase("%s: %llu %d %c", "6: 6557245727662973594 896373144 4", "\x01\x36\xb4\x8a\xe5\xe4\xeb\x8f\x82\x80\xb6\x01\xb0\xc6\xec\xd6\x06\x68"sv),
+TestCase("%s: %llu %d %c", "7: 5454275499512896944 1226045852 z", "\x01\x37\xe0\xf6\xa8\xf9\xe6\xaa\xbd\xb1\x97\x01\xb8\xe6\x9f\x91\x09\xf4\x01"sv),
+TestCase("%s: %llu %d %c", "8: 9044083069173435164 1766891050 '", "\x01\x38\xb8\xac\xa8\x93\xc8\xe8\x84\x83\xfb\x01\xd4\xf8\x84\x95\x0d\x4e"sv),
+TestCase("%s: %llu %d %c", "9: 5076270483973408909 977144804 ^", "\x01\x39\x9a\xa2\xc8\xd0\x84\xc7\xc4\xf2\x8c\x01\xc8\xaf\xf0\xa3\x07\xbc\x01"sv),
+TestCase("%s: %llu %d %c", "10: 6514086955327654755 2144124549 D", "\x02\x31\x30\xc6\xad\xfc\x84\xd4\xe4\xd7\xe6\xb4\x01\x8a\xfa\xe5\xfc\x0f\x88\x01"sv),
+TestCase("%s: %llu %d %c", "11: 4628058100229254151 1281962171 ", "\x02\x31\x31\x8e\xb0\xc5\x80\xcb\x94\x95\xba\x80\x01\xf6\xc2\xc9\xc6\x09\x40"sv),
+TestCase("%s: %llu %d %c", "12: 1076793340331741575 2057242876 Q", "\x02\x31\x32\x8e\xc6\xd6\xaf\xd0\xf5\xc4\xf1\x1d\xf8\xa3\xf8\xa9\x0f\xa2\x01"sv),
+TestCase("%s: %llu %d %c", "13: 5264779051526567427 890694850 .", "\x02\x31\x33\x86\xd8\xc5\xb2\xe8\xa9\xa0\x90\x92\x01\x84\xb3\xb7\xd1\x06\x5c"sv),
+TestCase("%s: %llu %d %c", "14: 6197432947465793532 1477386409 L", "\x02\x31\x34\xf8\xbf\x81\x90\xc5\x9b\xda\x81\xac\x01\xd2\x82\xf9\x80\x0b\x98\x01"sv),
+TestCase("%s: %llu %d %c", "15: 1939578830432974807 487012735 A", "\x02\x31\x35\xae\xdf\x99\x9d\xf0\xaa\xe2\xea\x35\xfe\xe5\xb9\xd0\x03\x82\x01"sv),
+TestCase("%s: %llu %d %c", "16: 4252410814844357301 2016407074 <", "\x02\x31\x36\xea\xaa\x9e\x9d\xe3\xc6\xcc\x83\x76\xc4\xb8\xff\x82\x0f\x78"sv),
+TestCase("%s: %llu %d %c", "17: 4095317955084962012 897441415 F", "\x02\x31\x37\xb8\xa3\xf2\xb1\xee\xfe\xbe\xd5\x71\x8e\xfa\xee\xd7\x06\x8c\x01"sv),
+TestCase("%s: %llu %d %c", "18: 8433702361086030159 2131660974 ~", "\x02\x31\x38\x9e\xc5\xbf\xde\xa3\xe7\xc3\x8a\xea\x01\xdc\xc2\xf4\xf0\x0f\xfc\x01"sv),
+TestCase("%s: %llu %d %c", "19: 5127196966915280688 1045937034 e", "\x02\x31\x39\xe0\xbc\x90\xa7\xbe\x9e\xbb\xa7\x8e\x01\x94\xee\xbd\xe5\x07\xca\x01"sv),
+TestCase("%s: %llu %d %c", "20: 8259453203008866922 1179139938 A", "\x02\x32\x30\xd4\xa9\xf2\xc4\xaf\xbb\xbc\x9f\xe5\x01\xc4\xfd\xc1\xe4\x08\x82\x01"sv),
+TestCase("%s: %llu %d %c", "21: 8275937406817640763 443381940 G", "\x02\x32\x31\xf6\x84\xdd\xcb\xa8\xce\x84\xda\xe5\x01\xe8\xe2\xeb\xa6\x03\x8e\x01"sv),
+TestCase("%s: %llu %d %c", "22: 997754361730893686 1007730161 e", "\x02\x32\x32\xec\x9d\x88\x81\xc8\x93\xde\xd8\x1b\xe2\xf7\x85\xc1\x07\xca\x01"sv),
+TestCase("%s: %llu %d %c", "23: 256781249510239018 1812585134 !", "\x02\x32\x33\xd4\xac\xcf\xa0\xf3\xcb\xa2\x90\x07\xdc\xea\xce\xc0\x0d\x42"sv),
+TestCase("%s: %llu %d %c", "24: 2332871340892345868 1139696535 2", "\x02\x32\x34\x98\xd8\x9e\xc9\xfb\x87\x83\xe0\x40\xae\x8e\xf3\xbe\x08\x64"sv),
+TestCase("%s: %llu %d %c", "25: 1987972021100915124 725058015 q", "\x02\x32\x35\xe8\xa6\x95\xd2\xa7\x81\xd9\x96\x37\xbe\x87\xbc\xb3\x05\xe2\x01"sv),
+TestCase("%s: %llu %d %c", "26: 5784824274936856659 746840193 y", "\x02\x32\x36\xa6\x81\xd8\xba\xeb\xc1\xe9\xc7\xa0\x01\x82\x82\x9f\xc8\x05\xf2\x01"sv),
+TestCase("%s: %llu %d %c", "27: 1164916386415315063 68230794 $", "\x02\x32\x37\xee\xa1\x88\xd5\x81\xd2\xce\xaa\x20\x94\xfa\x88\x41\x48"sv),
+TestCase("%s: %llu %d %c", "28: 3140932642079254647 430103045 r", "\x02\x32\x38\xee\xa1\x81\xa9\x97\xe6\xea\x96\x57\x8a\xe8\x96\x9a\x03\xe4\x01"sv),
+TestCase("%s: %llu %d %c", "29: 7940449326902609449 1006905623 N", "\x02\x32\x39\xd2\x98\xa8\xf0\xf5\xa5\x92\xb2\xdc\x01\xae\xa4\xa1\xc0\x07\x9c\x01"sv),
+TestCase("%s: %llu %d %c", "30: 5240496025593009868 112523603 (", "\x02\x33\x30\x98\xdb\x8a\xc5\xd6\xd7\xfd\xb9\x91\x01\xa6\xe5\xa7\x6b\x50"sv),
+TestCase("%s: %llu %d %c", "31: 5038839083535780307 849780316 ~", "\x02\x33\x31\xa6\xa7\x8a\xe8\xe2\xdc\xc6\xed\x8b\x01\xb8\xf9\xb4\xaa\x06\xfc\x01"sv),
+TestCase("%s: %llu %d %c", "32: 3630797309549363234 1877081269 B", "\x02\x33\x32\xc4\x80\x92\xf2\xd1\xba\x97\xe3\x64\xea\xf2\x8f\xfe\x0d\x84\x01"sv),
+TestCase("%s: %llu %d %c", "33: 3467724524658884800 1223515015 (", "\x02\x33\x33\x80\x83\xa0\xf3\xa2\xc5\xea\x9f\x60\x8e\xee\xea\x8e\x09\x50"sv),
+TestCase("%s: %llu %d %c", "34: 7712429266319135356 1217500709 l", "\x02\x33\x34\xf8\xc9\xba\xe4\xc6\xe3\x86\x88\xd6\x01\xca\xd8\x8c\x89\x09\xd8\x01"sv),
+TestCase("%s: %llu %d %c", "35: 3291981441856444913 1855334785 y", "\x02\x33\x35\xe2\xe7\xc0\x90\xc0\xeb\xbb\xaf\x5b\x82\xa6\xb1\xe9\x0d\xf2\x01"sv),
+TestCase("%s: %llu %d %c", "36: 4919847686728209916 715097219 :", "\x02\x33\x36\xf8\xd7\xb2\xc7\xf2\xd9\xe7\xc6\x88\x01\x86\x92\xfc\xa9\x05\x74"sv),
+TestCase("%s: %llu %d %c", "37: 6001314382948090489 1275341105 ]", "\x02\x33\x37\xf2\x89\x82\xd1\xd1\xe8\xf9\xc8\xa6\x01\xe2\xa4\xa1\xc0\x09\xba\x01"sv),
+TestCase("%s: %llu %d %c", "38: 3119268858057174477 692013498 2", "\x02\x33\x38\x9a\xb7\xfd\xb6\xfe\x9f\xef\xc9\x56\xf4\xa6\xfa\x93\x05\x64"sv),
+TestCase("%s: %llu %d %c", "39: 3141952960809904282 1723961557 I", "\x02\x33\x39\xb4\xa2\xfd\xa0\xc6\xe4\xba\x9a\x57\xaa\xc3\x8c\xec\x0c\x92\x01"sv),
+TestCase("%s: %llu %d %c", "40: 3962329461475475834 2013070724 s", "\x02\x34\x30\xf4\x85\xfc\xa4\xdc\xe9\x82\xfd\x6d\x88\x96\xe8\xff\x0e\xe6\x01"sv),
+TestCase("%s: %llu %d %c", "41: 6705183005166907218 2031346987 f", "\x02\x34\x31\xa4\xbd\xad\x8f\xdd\x9a\xcc\x8d\xba\x01\xd6\x94\x9f\x91\x0f\xcc\x01"sv),
+TestCase("%s: %llu %d %c", "42: 5275577307833715720 1461891095 B", "\x02\x34\x32\x90\xb0\xd2\xa8\x87\xe7\xce\xb6\x92\x01\xae\xc0\x95\xf2\x0a\x84\x01"sv),
+TestCase("%s: %llu %d %c", "43: 4999233507689781945 991216024 |", "\x02\x34\x33\xf2\xea\xa2\xf1\xeb\x98\xec\xe0\x8a\x01\xb0\x86\xa6\xb1\x07\xf8\x01"sv),
+TestCase("%s: %llu %d %c", "44: 2788213596089450528 1662775838 '", "\x02\x34\x34\xc0\x80\x9c\x8d\xc7\xdf\xdc\xb1\x4d\xbc\xc8\xdf\xb1\x0c\x4e"sv),
+TestCase("%s: %llu %d %c", "45: 1507235571856201971 690944234 &", "\x02\x34\x35\xe6\xe3\x92\xd5\x8a\xb1\xe3\xea\x29\xd4\xe3\xf7\x92\x05\x4c"sv),
+TestCase("%s: %llu %d %c", "46: 8120818212305491728 1991854726 U", "\x02\x34\x36\xa0\xdc\xee\xdf\xeb\xc8\xf8\xb2\xe1\x01\x8c\xaa\xca\xeb\x0e\xaa\x01"sv),
+TestCase("%s: %llu %d %c", "47: 850107129610567275 826171792 A", "\x02\x34\x37\xd6\xf9\xb7\x97\xc5\xfb\x97\xcc\x17\xa0\x86\xf3\x93\x06\x82\x01"sv),
+TestCase("%s: %llu %d %c", "48: 3619717899166040237 5848102 [", "\x02\x34\x38\xda\x82\xd2\xee\x89\x90\xe9\xbb\x64\xcc\xf0\xc9\x05\xb6\x01"sv),
+TestCase("%s: %llu %d %c", "49: 4766478454387430761 938318292 '", "\x02\x34\x39\xd2\xd5\x86\xf5\xcf\xb9\xf7\xa5\x84\x01\xa8\xe7\xec\xfe\x06\x4e"sv),
+TestCase("%s: %llu %d %c", "50: 7278057998974406311 2037680599 q", "\x02\x35\x30\xce\xba\xfa\xb8\x8e\xcc\xed\x80\xca\x01\xae\xa7\xa4\x97\x0f\xe2\x01"sv),
+TestCase("%s: %llu %d %c", "51: 3273441488341945355 1215440713 U", "\x02\x35\x31\x96\xb0\xae\x9a\x96\xec\xcc\xed\x5a\x92\x9d\x91\x87\x09\xaa\x01"sv),
+TestCase("%s: %llu %d %c", "52: 1352148195246416250 1195236094 E", "\x02\x35\x32\xf4\x85\xc9\xd5\xd3\xe7\xe5\xc3\x25\xfc\xeb\xee\xf3\x08\x8a\x01"sv),
+TestCase("%s: %llu %d %c", "53: 1676322249364352341 1204736997 m", "\x02\x35\x33\xaa\xb5\x91\xa7\x89\x8d\xbf\xc3\x2e\xca\xcf\xf6\xfc\x08\xda\x01"sv),
+TestCase("%s: %llu %d %c", "54: 2333671424671038513 1489186116 [", "\x02\x35\x34\xe2\xf0\xf9\x9f\xfc\xf2\xee\xe2\x40\x88\xb5\x99\x8c\x0b\xb6\x01"sv),
+TestCase("%s: %llu %d %c", "55: 4281900075145433102 1885273938 ~", "\x02\x35\x35\x9c\xc0\x98\xec\xd1\xdb\xae\xec\x76\xa4\xfd\xf7\x85\x0e\xfc\x01"sv),
+TestCase("%s: %llu %d %c", "56: 1034989118514460270 670362370 O", "\x02\x35\x36\xdc\xd9\x97\xb7\xd4\xc7\x82\xdd\x1c\x84\xac\xa7\xff\x04\x9e\x01"sv),
+TestCase("%s: %llu %d %c", "57: 2474224803315291647 852899224 h", "\x02\x35\x37\xfe\xd7\x8c\x81\xb0\x96\x9b\xd6\x44\xb0\xd6\xb1\xad\x06\xd0\x01"sv),
+TestCase("%s: %llu %d %c", "58: 6086297047179889204 792444130 G", "\x02\x35\x38\xe8\xa8\xd4\xea\xde\xba\xef\xf6\xa8\x01\xc4\xf3\xdd\xf3\x05\x8e\x01"sv),
+TestCase("%s: %llu %d %c", "59: 4129732865765686804 1968869102 U", "\x02\x35\x39\xa8\xf8\xc1\x93\xa8\x8a\xe1\xcf\x72\xdc\xbb\xd4\xd5\x0e\xaa\x01"sv),
+TestCase("%s: %llu %d %c", "60: 8297459921109698914 642375500 A", "\x02\x36\x30\xc4\xa5\xc6\xfc\xc3\xf5\xbf\xa6\xe6\x01\x98\xfd\xce\xe4\x04\x82\x01"sv),
+TestCase("%s: %llu %d %c", "61: 8848352886883606505 278420198 )", "\x02\x36\x31\xd2\xff\xfe\xdf\xbd\x84\xd5\xcb\xf5\x01\xcc\xeb\xc2\x89\x02\x52"sv),
+TestCase("%s: %llu %d %c", "62: 639367729021931858 519678154 p", "\x02\x36\x32\xa4\xc5\x9a\x93\xfc\xe3\xbe\xdf\x11\x94\xa3\xcd\xef\x03\xe0\x01"sv),
+TestCase("%s: %llu %d %c", "63: 4168177859746898377 293410050 {", "\x02\x36\x33\x92\x87\xe5\xad\xbe\xeb\xab\xd8\x73\x84\xd4\xe8\x97\x02\xf6\x01"sv),
+TestCase("%s: %llu %d %c", "64: 8933020158994285890 1727299316 n", "\x02\x36\x34\x84\x95\x90\xc4\xac\xa0\xbb\xf8\xf7\x01\xe8\xfb\xa3\xef\x0c\xdc\x01"sv),
+TestCase("%s: %llu %d %c", "65: 2904831533300168615 1786519136 V", "\x02\x36\x35\xce\xde\xad\xab\xf3\xb8\x84\xd0\x50\xc0\xf9\xe0\xa7\x0d\xac\x01"sv),
+TestCase("%s: %llu %d %c", "66: 625118648746706096 1778893413 Q", "\x02\x36\x36\xe0\x92\xa8\x90\xab\x86\xef\xac\x11\xca\x89\xbe\xa0\x0d\xa2\x01"sv),
+TestCase("%s: %llu %d %c", "67: 3317249430279903162 738620162 '", "\x02\x36\x37\xf4\xde\x9a\xb3\x80\xb2\x9e\x89\x5c\x84\xcc\xb3\xc0\x05\x4e"sv),
+TestCase("%s: %llu %d %c", "68: 2529759202900350655 1638933594 =", "\x02\x36\x38\xfe\xda\xb4\x82\xf0\xa5\xc1\x9b\x46\xb4\x91\x81\x9b\x0c\x7a"sv),
+TestCase("%s: %llu %d %c", "69: 4162847733300040370 641234596 `", "\x02\x36\x39\xe4\xca\x9c\xd6\xa5\xfd\xb3\xc5\x73\xc8\xda\xc3\xe3\x04\xc0\x01"sv),
+TestCase("%s: %llu %d %c", "70: 8048952107112912959 1411687739 _", "\x02\x37\x30\xfe\xe0\xc6\xe9\xfa\xd2\xcf\xb3\xdf\x01\xf6\x94\xa5\xc2\x0a\xbe\x01"sv),
+TestCase("%s: %llu %d %c", "71: 1860779983819210237 1258094565 Q", "\x02\x37\x31\xfa\xf7\x9b\xcb\xaa\xe2\xe8\xd2\x33\xca\xff\xe7\xaf\x09\xa2\x01"sv),
+TestCase("%s: %llu %d %c", "72: 133586295505940516 1385623828 t", "\x02\x37\x32\xc8\xc0\xef\xa7\xe5\x81\xcc\xda\x03\xa8\xc4\xb7\xa9\x0a\xe8\x01"sv),
+TestCase("%s: %llu %d %c", "73: 476590412526583632 1426152233 ", "\x02\x37\x33\xa0\xfd\x87\xa0\x90\x9d\x98\x9d\x0d\xd2\xec\x8a\xd0\x0a\x12"sv),
+TestCase("%s: %llu %d %c", "74: 7182329075191030773 870570930 }", "\x02\x37\x34\xea\x8f\xc5\xab\xb0\x8f\xe1\xac\xc7\x01\xe4\xee\x9e\xbe\x06\xfa\x01"sv),
+TestCase("%s: %llu %d %c", "75: 6781911602451840052 1186936344 Z", "\x02\x37\x35\xe8\xf0\xef\xa3\xf3\xaa\x98\x9e\xbc\x01\xb0\xd8\xf9\xeb\x08\xb4\x01"sv),
+TestCase("%s: %llu %d %c", "76: 2755779697069684711 1068026786 W", "\x02\x37\x36\xce\xdf\x93\xb1\x94\xc2\xbf\xbe\x4c\xc4\xae\xc6\xfa\x07\xae\x01"sv),
+TestCase("%s: %llu %d %c", "77: 7605101363071432517 182922063 S", "\x02\x37\x37\x8a\xdd\xb1\xab\xad\xd9\xdf\x8a\xd3\x01\x9e\xad\xb9\xae\x01\xa6\x01"sv),
+TestCase("%s: %llu %d %c", "78: 8440954818933111077 809999951 +", "\x02\x37\x38\xca\xe4\x89\xba\xf1\xeb\xa5\xa4\xea\x01\x9e\xf9\xbc\x84\x06\x56"sv),
+TestCase("%s: %llu %d %c", "79: 177390365453428882 521931632 *", "\x02\x37\x39\xa4\xa2\xed\xf6\x9e\xe6\x9b\xf6\x04\xe0\xad\xe0\xf1\x03\x54"sv),
+TestCase("%s: %llu %d %c", "80: 2690515256289614444 1230288283 #", "\x02\x38\x30\xd8\xe9\xca\x9e\xdc\xd7\xd0\xd6\x4a\xb6\xd6\xa5\x95\x09\x46"sv),
+TestCase("%s: %llu %d %c", "81: 7345613454774364586 1636244573 z", "\x02\x38\x31\xd4\xa6\xbf\xd2\x96\xa1\xee\xf0\xcb\x01\xba\xf1\xb8\x98\x0c\xf4\x01"sv),
+TestCase("%s: %llu %d %c", "82: 6370586723714568954 1353978889 X", "\x02\x38\x32\xf4\xcb\x9c\xdd\xea\xb7\xef\xe8\xb0\x01\x92\xd0\xa0\x8b\x0a\xb0\x01"sv),
+TestCase("%s: %llu %d %c", "83: 731720363738568228 1838212903 J", "\x02\x38\x33\xc8\xd8\xf4\xb8\xe8\xf2\xcb\xa7\x14\xce\x9c\x87\xd9\x0d\x94\x01"sv),
+TestCase("%s: %llu %d %c", "84: 9161537274294991160 1309917278 u", "\x02\x38\x34\xf0\xe4\xd3\xed\x8d\xe7\xa8\xa4\xfe\x01\xbc\x81\x9e\xe1\x09\xea\x01"sv),
+TestCase("%s: %llu %d %c", "85: 1558715097695343451 97413929 Z", "\x02\x38\x35\xb6\xcd\xda\x82\xef\xc7\xd5\xa1\x2b\xd2\xac\xf3\x5c\xb4\x01"sv),
+TestCase("%s: %llu %d %c", "86: 366808135709449019 1760066788 z", "\x02\x38\x36\xf6\xec\xfa\xda\xca\x83\x95\x97\x0a\xc8\xf3\xc3\x8e\x0d\xf4\x01"sv),
+TestCase("%s: %llu %d %c", "87: 891656539731055874 742154760 1", "\x02\x38\x37\x84\xd4\x96\x85\xc2\xb9\xe6\xdf\x18\x90\x88\xe3\xc3\x05\x62"sv),
+TestCase("%s: %llu %d %c", "88: 3732114429930137548 1005229045 2", "\x02\x38\x38\x98\x9f\xc3\x91\x9d\x93\x91\xcb\x67\xea\xcf\xd4\xbe\x07\x64"sv),
+TestCase("%s: %llu %d %c", "89: 8113837181842537067 1082098316 G", "\x02\x38\x39\xd6\xe9\xa5\xd1\xb1\xfb\x91\x9a\xe1\x01\x98\x8a\xfc\x87\x08\x8e\x01"sv),
+TestCase("%s: %llu %d %c", "90: 2770024080011047550 1330821140 N", "\x02\x39\x30\xfc\xc9\xb1\xc5\xaf\x8e\x8d\xf1\x4c\xa8\xe0\x95\xf5\x09\x9c\x01"sv),
+TestCase("%s: %llu %d %c", "91: 4697198541546045125 2128296134 x", "\x02\x39\x31\x8a\xfb\xad\xc5\xf9\xcb\xe6\xaf\x82\x01\x8c\xe3\xd9\xed\x0f\xf0\x01"sv),
+TestCase("%s: %llu %d %c", "92: 1018100739973888361 570654735 w", "\x02\x39\x32\xd2\xe5\xa5\xfb\xd2\xce\x82\xa1\x1c\x9e\x80\x9c\xa0\x04\xee\x01"sv),
+TestCase("%s: %llu %d %c", "93: 4859753067646279323 676264182 W", "\x02\x39\x33\xb6\xfa\xde\xd2\xd7\xea\xa7\xf1\x86\x01\xec\xe3\xf7\x84\x05\xae\x01"sv),
+TestCase("%s: %llu %d %c", "94: 549670438443165922 132670573 t", "\x02\x39\x34\xc4\xa3\x81\xaf\xbf\x96\xe9\xa0\x0f\xda\x91\xc3\x7e\xe8\x01"sv),
+TestCase("%s: %llu %d %c", "95: 3444356383434812300 501045255 2", "\x02\x39\x35\x98\xae\xd9\xa7\xb7\xf8\xe7\xcc\x5f\x8e\xe0\xea\xdd\x03\x64"sv),
+TestCase("%s: %llu %d %c", "96: 2011546512680833085 1077631988 Z", "\x02\x39\x36\xfa\xc0\xa2\x98\xa9\xb9\xb9\xea\x37\xe8\xef\xda\x83\x08\xb4\x01"sv),
+TestCase("%s: %llu %d %c", "97: 4692262078416922470 1829700092 '", "\x02\x39\x37\xcc\x9d\x9d\x91\xfd\xdf\xa1\x9e\x82\x01\xf8\x87\xf8\xd0\x0d\x4e"sv),
+TestCase("%s: %llu %d %c", "98: 4919691286333696629 1322133483 l", "\x02\x39\x38\xea\xa9\x89\xe8\x98\xca\xa0\xc6\x88\x01\xd6\x9f\xf1\xec\x09\xd8\x01"sv),
+TestCase("%s: %llu %d %c", "99: 1241746909443220722 1232171669 L", "\x02\x39\x39\xe4\x93\xeb\xe4\x85\x8f\xc9\xbb\x22\xaa\xca\x8b\x97\x09\x98\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "0: 1941514050836526867 0x 21293775377EEBB901 101", "\x01\x30\xa6\xdc\xd3\xe8\xc3\xaf\xd2\xf1\x35\x82\xe4\xdd\xee\x0f\x82\xe4\xdd\xee\x0f\xca\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "1: 3509957387132194637 0x 19171154647244DC48 33", "\x01\x31\x9a\xbd\xae\xcc\xb4\xe9\xef\xb5\x61\x90\xf1\xa6\xa4\x0e\x90\xf1\xa6\xa4\x0e\x42"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "2: 8024983545482782384 0x 15299301625B30E1B2 46", "\x01\x32\xe0\x9a\xd9\xcd\x86\x81\xbc\xde\xde\x01\xe4\x86\x87\xb3\x0b\xe4\x86\x87\xb3\x0b\x5c"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "3: 5058360797369425151 0x 10725304213FED83F5 104", "\x01\x33\xfe\xf3\xba\x8f\x89\x96\xf4\xb2\x8c\x01\xea\x8f\xec\xfe\x07\xea\x8f\xec\xfe\x07\xd0\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "4: 6529118911438932678 0x 9786230773A549A65 105", "\x01\x34\x8c\xab\xd2\xc9\xd4\xc3\x8b\x9c\xb5\x01\xca\xe9\xa4\xa5\x07\xca\xe9\xa4\xa5\x07\xd2\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "5: 4463425652718897265 0x 17497759010AFE9F 73", "\x01\x35\xe2\xa1\xe8\xfa\xf3\xfe\xa2\xf1\x7b\xbe\xfa\xd7\x10\xbe\xfa\xd7\x10\x92\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "6: 5434900075521908661 0x 123595425449AB2A4E 45", "\x01\x36\xea\xfe\xd0\xf3\xd8\xb4\xd2\xec\x96\x01\x9c\xa9\xd9\x9a\x09\x9c\xa9\xd9\x9a\x09\x5a"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "7: 715267470600062067 0x 120413030347C591FF 114", "\x01\x37\xe6\x91\x92\x88\xb2\xfe\x91\xed\x13\xfe\xc7\xac\xfc\x08\xfe\xc7\xac\xfc\x08\xe4\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "8: 3926011308434788247 0x 12666770164B7FF518 85", "\x01\x38\xae\xbe\xe0\xf6\xfd\x9e\xff\xfb\x6c\xb0\xd4\xff\xb7\x09\xb0\xd4\xff\xb7\x09\xaa\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "9: 1478056893002169192 0x 7358327362BDBEAA0 35", "\x01\x39\xd0\xed\xb1\x93\xb8\xba\x8e\x83\x29\xc0\xaa\xdf\xbd\x05\xc0\xaa\xdf\xbd\x05\x46"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "10: 7128491551588143718 0x 1717398262665D6AF6 94", "\x02\x31\x30\xcc\xc9\xc6\xcc\x87\xd3\xbe\xed\xc5\x01\xec\xab\xeb\xe5\x0c\xec\xab\xeb\xe5\x0c\xbc\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "11: 193231861624509923 0x 4552840211B231535 41", "\x02\x31\x31\xc6\xd7\xd4\xf6\xd6\xd6\xbf\xae\x05\xea\xd4\x98\xb2\x03\xea\xd4\x98\xb2\x03\x52"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "12: 4888125773563526975 0x 1378744602522DF91A 110", "\x02\x31\x32\xfe\xfc\xfb\xea\xc3\x9f\x8e\xd6\x87\x01\xb4\xe4\xef\xa2\x0a\xb4\xe4\xef\xa2\x0a\xdc\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "13: 7673779374634366645 0x 1131769900ACB1C3 60", "\x02\x31\x33\xea\xda\xe7\xdc\xe1\xeb\xde\xfe\xd4\x01\x86\xc7\xe5\x0a\x86\xc7\xe5\x0a\x78"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "14: 5854341954344571365 0x 108816167440DC078A 107", "\x02\x31\x34\xca\xc7\xfe\xe5\xac\xbf\xe6\xbe\xa2\x01\x94\x9e\xe0\x8d\x08\x94\x9e\xe0\x8d\x08\xd6\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "15: 2326184191199164007 0x 8137466533080C9DD 82", "\x02\x31\x35\xce\xd9\xf1\xe8\xcf\x8c\xa2\xc8\x40\xba\xa7\x86\x88\x06\xba\xa7\x86\x88\x06\xa4\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "16: 1424330851786829530 0x 1208787250480CA132 57", "\x02\x31\x36\xb4\xbb\x91\xf6\x9f\xd7\x9e\xc4\x27\xe4\x84\xe5\x80\x09\xe4\x84\xe5\x80\x09\x72"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "17: 6389735809259880362 0x 18144454646C263D98 42", "\x02\x31\x37\xd4\xfe\xb9\xf8\xa7\xb7\xf3\xac\xb1\x01\xb0\xf6\xb1\xc2\x0d\xb0\xf6\xb1\xc2\x0d\x54"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "18: 4564825427452770815 0x 2195709200D1662E8 62", "\x02\x31\x38\xfe\xe7\xf1\x8f\xce\xa2\xc2\xd9\x7e\xd0\x8b\xb3\xd1\x01\xd0\x8b\xb3\xd1\x01\x7c"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "19: 1621021906259841134 0x 177653957669E3D7B8 93", "\x02\x31\x39\xdc\xc1\xf4\xfb\xb9\xb5\x83\xff\x2c\xf0\xde\x9e\x9e\x0d\xf0\xde\x9e\x9e\x0d\xba\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "20: 8025284571884195522 0x 13595901875109B32B 94", "\x02\x32\x30\x84\xbb\xa4\x89\x8c\xf3\xc4\xdf\xde\x01\xd6\xcc\xcd\x90\x0a\xd6\xcc\xcd\x90\x0a\xbc\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "21: 1310759272316304459 0x 85298242232D77A96 119", "\x02\x32\x31\x96\xf1\xb3\xc7\xa1\xa7\xe0\xb0\x24\xac\xea\xbb\xad\x06\xac\xea\xbb\xad\x06\xee\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "22: 3661335273992329249 0x 825503981313430ED 105", "\x02\x32\x32\xc2\xe0\xdc\x9e\x8f\xc2\xd6\xcf\x65\xda\xc3\xa1\x93\x06\xda\xc3\xa1\x93\x06\xd2\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "23: 4908269103466540013 0x 20984305627D138262 115", "\x02\x32\x33\xda\xaf\xaf\xe2\xd2\xaf\xd6\x9d\x88\x01\xc4\x89\x9c\xd1\x0f\xc4\x89\x9c\xd1\x0f\xe6\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "24: 6541851858139706195 0x 148818392458B3E274 57", "\x02\x32\x34\xa6\xcd\xab\x9a\xda\xe6\xa9\xc9\xb5\x01\xe8\x89\x9f\x8b\x0b\xe8\x89\x9f\x8b\x0b\x72"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "25: 4510904210489015127 0x 1191906481470B0CB1 71", "\x02\x32\x35\xae\xed\x97\xa0\xd9\xde\xf9\x99\x7d\xe2\xb2\xd8\xf0\x08\xe2\xb2\xd8\xf0\x08\x8e\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "26: 1751202991906984173 0x 10412118623E0FA1D6 115", "\x02\x32\x36\xda\xc3\xbc\xe3\xf8\xf5\xc2\xcd\x30\xac\x87\xfd\xe0\x07\xac\x87\xfd\xe0\x07\xe6\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "27: 7912743439818658557 0x 15466272805C2FA8D0 112", "\x02\x32\x37\xfa\xcb\x8c\x93\x85\x8f\xdb\xcf\xdb\x01\xa0\xa3\xfd\xc2\x0b\xa0\xa3\xfd\xc2\x0b\xe0\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "28: 7865046769076850396 0x 95679371239078370 56", "\x02\x32\x38\xb8\xbb\xc6\xca\xb3\x97\xa1\xa6\xda\x01\xe0\x8d\xbc\x90\x07\xe0\x8d\xbc\x90\x07\x70"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "29: 436719658465345699 0x 5945969062370D42A 43", "\x02\x32\x39\xc6\x82\x8e\xd4\xe5\x8d\xc5\x8f\x0c\xd4\xd0\x86\xb7\x04\xd4\xd0\x86\xb7\x04\x56"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "30: 6212224561138197241 0x 1234914600BC6EDA 87", "\x02\x33\x30\xf2\x8b\xcb\xf9\xe2\xd4\xa0\xb6\xac\x01\xb4\xbb\xe3\x0b\xb4\xbb\xe3\x0b\xae\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "31: 5419660422277913790 0x 177818160669FCE5E6 39", "\x02\x33\x31\xfc\xf2\xf2\x91\x90\x9c\xc0\xb6\x96\x01\xcc\x97\xe7\x9f\x0d\xcc\x97\xe7\x9f\x0d\x4e"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "32: 1955771002711010609 0x 175837101468CE9CC6 114", "\x02\x33\x32\xe2\xb4\xcb\xe9\xac\xd7\xa5\xa4\x36\x8c\xf3\xf4\x8c\x0d\x8c\xf3\xf4\x8c\x0d\xe4\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "33: 4550116504720438569 0x 1763101400A82477C 115", "\x02\x33\x33\xd2\xd4\x87\xe8\xcf\xb6\xa1\xa5\x7e\xf8\x9d\x92\xa8\x01\xf8\x9d\x92\xa8\x01\xe6\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "34: 4684454901623556905 0x 4891671541D281932 106", "\x02\x33\x34\xd2\xac\xe0\x99\xbd\xba\xc3\x82\x82\x01\xe4\xe4\xc0\xd2\x03\xe4\xe4\xc0\xd2\x03\xd4\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "35: 6234369280162760629 0x 39351496417748FD4 46", "\x02\x33\x35\xea\x9e\xac\xeb\x88\xf5\xf6\x84\xad\x01\xa8\xbf\xa4\xf7\x02\xa8\xbf\xa4\xf7\x02\x5c"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "36: 8944940508995398022 0x 137633313452092D4E 45", "\x02\x33\x36\x8c\xa6\xe1\xfa\x84\x80\xe8\xa2\xf8\x01\x9c\xb5\xc9\xa0\x0a\x9c\xb5\xc9\xa0\x0a\x5a"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "37: 4077388965675931531 0x 2899199101147D3A6 47", "\x02\x33\x37\x96\xde\xb0\xe3\xa9\xea\xe5\x95\x71\xcc\xce\xbe\x94\x02\xcc\xce\xbe\x94\x02\x5e"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "38: 8856210144434134086 0x 1990062769769DF2B1 83", "\x02\x33\x38\x8c\xf1\xa8\xb3\x88\x8d\xca\xe7\xf5\x01\xe2\xca\xef\xe9\x0e\xe2\xca\xef\xe9\x0e\xa6\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "39: 6269182192420305924 0x 15904663025ECC96FE 114", "\x02\x33\x39\x88\xf0\xd5\xb6\xa1\xff\xcd\x80\xae\x01\xfc\xdb\xe4\xec\x0b\xfc\xdb\xe4\xec\x0b\xe4\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "40: 3168030316008523010 0x 560637674216AA6EA 54", "\x02\x34\x30\x84\x94\xda\xf5\xb2\xb2\x8d\xf7\x57\xd4\x9b\xd5\x96\x04\xd4\x9b\xd5\x96\x04\x6c"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "41: 7828361129715530032 0x 120199790547A50851 125", "\x02\x34\x31\xe0\xe4\xdb\xe2\x92\xbe\xf6\xa3\xd9\x01\xa2\xa1\xa8\xfa\x08\xa2\xa1\xa8\xfa\x08\xfa\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "42: 4684430121285811332 0x 1645956019621B4BB3 49", "\x02\x34\x32\x88\xf2\x8b\xa6\x89\x98\xb8\x82\x82\x01\xe6\xae\xda\xa1\x0c\xe6\xae\xda\xa1\x0c\x62"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "43: 1094908319066476988 0x 200784098877AD38DC 34", "\x02\x34\x33\xf8\xc6\x9e\xb9\x95\xd4\xf2\xb1\x1e\xb8\xe3\xe9\xfa\x0e\xb8\xe3\xe9\xfa\x0e\x44"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "44: 4188361875415362989 0x 193392125773454BE9 109", "\x02\x34\x34\xda\xb6\xdf\xee\xe9\xbb\x86\xa0\x74\xd2\xaf\xaa\xb4\x0e\xd2\xaf\xaa\xb4\x0e\xda\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "45: 6350883509347762576 0x 147404369957DC1F33 76", "\x02\x34\x35\xa0\x86\xfa\x88\xe8\xb9\xef\xa2\xb0\x01\xe6\xfc\xe0\xfd\x0a\xe6\xfc\xe0\xfd\x0a\x98\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "46: 3843711385867305612 0x 17834016666A4C8CC2 61", "\x02\x34\x36\x98\x8a\xf6\xdb\xff\xc8\xcd\xd7\x6a\x84\xb3\xe4\xa4\x0d\x84\xb3\xe4\xa4\x0d\x7a"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "47: 5535789636778214509 0x 1184586398469B5A9E 84", "\x02\x34\x37\xda\x91\xa2\xb7\x88\xd6\x89\xd3\x99\x01\xbc\xea\xda\xe9\x08\xbc\xea\xda\xe9\x08\xa8\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "48: 8891437578007474517 0x 1695359970A1AE9FD 111", "\x02\x34\x38\xaa\xc5\x8f\xbb\xc9\xd7\xdd\xe4\xf6\x01\xfa\xa7\xd7\xa1\x01\xfa\xa7\xd7\xa1\x01\xde\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "49: 2483479174720276235 0x 14806493005840EA54 82", "\x02\x34\x39\x96\x9c\xc2\x84\xd8\xc9\x8b\xf7\x44\xa8\xa9\x87\x84\x0b\xa8\xa9\x87\x84\x0b\xa4\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "50: 6416832625845083020 0x 13297388064F423436 87", "\x02\x35\x30\x98\xee\x90\xea\xcf\xd1\x95\x8d\xb2\x01\xec\xd0\x91\xf4\x09\xec\xd0\x91\xf4\x09\xae\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "51: 3612059979445064775 0x 20936993057CCB50E9 117", "\x02\x35\x31\x8e\xd1\xd2\xe3\xbd\xda\xce\xa0\x64\xd2\xc3\xda\xcc\x0f\xd2\xc3\xda\xcc\x0f\xea\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "52: 8599514281591661754 0x 1468170900E0066D 118", "\x02\x35\x32\xf4\x92\xdb\xb2\xa9\xab\xce\xd7\xee\x01\xda\x99\x80\x0e\xda\x99\x80\x0e\xec\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "53: 1367741239369008397 0x 531248300510FE3 73", "\x02\x35\x33\x9a\x94\xf0\xd3\xa3\xda\x98\xfb\x25\xc6\xbf\x88\x05\xc6\xbf\x88\x05\x92\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "54: 5473904087316166353 0x 5067733201E34BF48 68", "\x02\x35\x34\xa2\xdb\xcf\xb0\xfe\xb0\x9b\xf7\x97\x01\x90\xfd\xa5\xe3\x03\x90\xfd\xa5\xe3\x03\x88\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "55: 1822527202856628083 0x 15610026845D0B02BC 58", "\x02\x35\x35\xe6\xbd\x8b\xfc\xb6\xb5\xf5\xca\x32\xf8\x8a\xd8\xd0\x0b\xf8\x8a\xd8\xd0\x0b\x74"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "56: 3032341809633898873 0x 10226365813CF43225 113", "\x02\x35\x36\xf2\xc5\xd9\x8c\xea\xb3\x85\x95\x54\xca\xc8\xa1\xcf\x07\xca\xc8\xa1\xcf\x07\xe2\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "57: 6856757614633255152 0x 8231254204E7FD5E 91", "\x02\x35\x37\xe0\x83\xdc\xb2\x99\xae\x8c\xa8\xbe\x01\xbc\xf5\xbf\x4e\xbc\xf5\xbf\x4e\xb6\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "58: 3224702228554841730 0x 189119246370B94E8F 92", "\x02\x35\x38\x84\xfa\xdf\xf6\xee\xe5\xb8\xc0\x59\x9e\xba\xca\x8b\x0e\x9e\xba\xca\x8b\x0e\xb8\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "59: 3988658283410987938 0x 176694389369516C95 34", "\x02\x35\x39\xc4\xee\x8c\x92\xee\xe4\xc7\xda\x6e\xaa\xb2\x8b\x95\x0d\xaa\xb2\x8b\x95\x0d\x44"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "60: 72920210115635164 0x 1901935881715D3D09 71", "\x02\x36\x30\xb8\x9f\x9d\xce\xee\xa2\x88\x83\x02\x92\xf4\xe9\x95\x0e\x92\xf4\xe9\x95\x0e\x8e\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "61: 4655403017017936000 0x 19053934447191FF24 48", "\x02\x36\x31\x80\xa2\x96\xba\x9d\x98\xa8\x9b\x81\x01\xc8\xfc\x8f\x99\x0e\xc8\xfc\x8f\x99\x0e\x60"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "62: 288952216557681443 0x 39039797517450017 85", "\x02\x36\x32\xc6\xec\xdb\x8f\xd2\xa0\xc8\x82\x08\xae\x80\xa8\xf4\x02\xae\x80\xa8\xf4\x02\xaa\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "63: 3219359012813218523 0x 21243024217E9E4855 51", "\x02\x36\x33\xb6\x8b\xf9\xb2\xe3\xfd\xba\xad\x59\xaa\xa1\xf2\xe9\x0f\xaa\xa1\xf2\xe9\x0f\x66"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "64: 7496923004744506260 0x 65433043327004A41 39", "\x02\x36\x34\xa8\xee\x80\xe4\x93\xf0\xb5\x8a\xd0\x01\x82\xa9\x82\xf0\x04\x82\xa9\x82\xf0\x04\x4e"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "65: 5886875793686089828 0x 2428972020E7A5132 73", "\x02\x36\x35\xc8\xc1\x8a\xf8\x84\x96\xb1\xb2\xa3\x01\xe4\xc4\xd2\xe7\x01\xe4\xc4\xd2\xe7\x01\x92\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "66: 8352418169916257811 0x 157093077095D0CD5 107", "\x02\x36\x36\xa6\xd8\x83\x96\xd2\x84\xe0\xe9\xe7\x01\xaa\xb3\xe8\x95\x01\xaa\xb3\xe8\x95\x01\xd6\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "67: 6425607598961991251 0x 142445717254E77DD4 102", "\x02\x36\x37\xa6\xf9\xf0\xa2\xa4\x84\xac\xac\xb2\x01\xa8\xf7\xbb\xce\x0a\xa8\xf7\xbb\xce\x0a\xcc\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "68: 2394571431170796410 0x 21402538797F91AEB7 105", "\x02\x36\x38\xf4\xad\xdf\x83\xf3\x81\x9d\xbb\x42\xee\xba\x8d\xf9\x0f\xee\xba\x8d\xf9\x0f\xd2\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "69: 891164515808164858 0x 10639639773F6ACD49 125", "\x02\x36\x39\xf4\xef\xed\x92\xfa\xd9\x86\xde\x18\x92\xb5\xd6\xf6\x07\x92\xb5\xd6\xf6\x07\xfa\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "70: 4245075167591901055 0x 21000887517D2CCFAF 64", "\x02\x37\x30\xfe\x9d\xad\x92\xf4\xd7\xc4\xe9\x75\xde\xbe\xe6\xd2\x0f\xde\xbe\xe6\xd2\x0f\x80\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "71: 979687222629424543 0x 958033112391A6CD8 45", "\x02\x37\x31\xbe\xf6\x86\xfd\xd3\x95\xc6\x98\x1b\xb0\xb3\xd3\x91\x07\xb0\xb3\xd3\x91\x07\x5a"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "72: 2602113737417537288 0x 93725236837DD5610 105", "\x02\x37\x32\x90\xdc\xfd\xfe\x8a\xaa\xc8\x9c\x48\xa0\xd8\xea\xfd\x06\xa0\xd8\xea\xfd\x06\xd2\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "73: 3036291044299697402 0x 2492532650EDB4D91 53", "\x02\x37\x33\xf4\xb3\xf7\xa2\xc5\xa7\x89\xa3\x54\xa2\xb6\xda\xed\x01\xa2\xb6\xda\xed\x01\x6a"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "74: 5631405388638357750 0x 1248216000BE7670 56", "\x02\x37\x34\xec\xa3\x94\x80\xaa\xd7\xe2\xa6\x9c\x01\xe0\xd9\xf3\x0b\xe0\xd9\xf3\x0b\x70"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "75: 6477169593541844553 0x 41341392918A43229 54", "\x02\x37\x35\x92\xf9\xae\xd0\xb0\xdb\xc3\xe3\xb3\x01\xd2\xc8\xa1\x8a\x03\xd2\xc8\xa1\x8a\x03\x6c"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "76: 720129488086281194 0x 15625894285D2338F4 83", "\x02\x37\x36\xd4\xaf\xd5\x9a\x87\xfd\xb4\xfe\x13\xe8\xe3\x99\xd2\x0b\xe8\xe3\x99\xd2\x0b\xa6\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "77: 7513165068981918756 0x 694699741296846DD 57", "\x02\x37\x37\xc8\xf0\xd2\xb8\xdc\xf4\x8f\xc4\xd0\x01\xba\x9b\xc2\x96\x05\xba\x9b\xc2\x96\x05\x72"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "78: 3737216259576487553 0x 5295090401F8FAAB0 121", "\x02\x37\x38\x82\xfa\xf6\x8d\xe6\x98\xa1\xdd\x67\xe0\xaa\xfd\xf8\x03\xe0\xaa\xfd\xf8\x03\xf2\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "79: 3565198557459535882 0x 86912895033CDDAF6 125", "\x02\x37\x39\x94\xe0\xe2\xb1\xdc\xcc\x90\xfa\x62\xec\xeb\xee\xbc\x06\xec\xeb\xee\xbc\x06\xfa\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "80: 5371965991102827603 0x 11276814506B8B491 120", "\x02\x38\x30\xa6\xb1\xce\xd2\xec\xa6\x87\x8d\x95\x01\xa2\xd2\xc5\x6b\xa2\xd2\xc5\x6b\xf0\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "81: 5776656679787538668 0x 18268958606CE437F4 72", "\x02\x38\x31\xd8\xa3\xfd\xc1\x9d\xa9\xe7\xaa\xa0\x01\xe8\xdf\xa1\xce\x0d\xe8\xdf\xa1\xce\x0d\x90\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "82: 749331093609658474 0x 96722711439A6B6EA 32", "\x02\x38\x32\xd4\xe1\x91\xdc\x9a\xaa\x94\xe6\x14\xd4\xdb\xb5\x9a\x07\xd4\xdb\xb5\x9a\x07\x40"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "83: 8161063962699005564 0x 86834268433C1DB9C 80", "\x02\x38\x33\xf8\xf9\xd3\xde\xea\x9b\xf6\xc1\xe2\x01\xb8\xee\x8e\xbc\x06\xb8\xee\x8e\xbc\x06\xa0\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "84: 3621720449882747425 0x 85086863932B7399F 122", "\x02\x38\x34\xc2\xc8\xbb\xb0\xef\xe3\xf7\xc2\x64\xbe\xe6\xb9\xab\x06\xbe\xe6\xb9\xab\x06\xf4\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "85: 8747087227033699739 0x 18257513516CD2C137 101", "\x02\x38\x35\xb6\x96\x9a\xab\xa5\xdf\xf2\xe3\xf2\x01\xee\x84\x96\xcd\x0d\xee\x84\x96\xcd\x0d\xca\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "86: 8253451662283365177 0x 7840661982EBBE696 79", "\x02\x38\x36\xf2\xec\x8e\xd4\xdc\xa3\x93\x8a\xe5\x01\xac\x9a\xdf\xeb\x05\xac\x9a\xdf\xeb\x05\x9e\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "87: 134795648044988988 0x 5147834581EAEF8E2 105", "\x02\x38\x37\xf8\xe8\xeb\xef\xaf\xfb\xf1\xde\x03\xc4\xe3\xf7\xea\x03\xc4\xe3\xf7\xea\x03\xd2\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "88: 421750780720578633 0x 137027843451ACCA22 65", "\x02\x38\x38\x92\x81\xc0\xc9\xb7\x86\xae\xda\x0b\xc4\xa8\xe6\x9a\x0a\xc4\xa8\xe6\x9a\x0a\x82\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "89: 8457438023241447265 0x 200824394177B35EE5 47", "\x02\x38\x39\xc2\xdd\x81\xd3\xd3\xc4\xed\xde\xea\x01\xca\xfb\x9a\xfb\x0e\xca\xfb\x9a\xfb\x0e\x5e"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "90: 4522738759120336903 0x 6219044132511821D 72", "\x02\x39\x30\x8e\xe0\x91\xd4\x8b\xbc\xff\xc3\x7d\xba\x88\x8c\xd1\x04\xba\x88\x8c\xd1\x04\x90\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "91: 3110596655769706088 0x 167356506563C09389 68", "\x02\x39\x31\xd0\xf9\xde\xaa\xb0\xcb\x87\xab\x56\x92\xce\x84\xbc\x0c\x92\xce\x84\xbc\x0c\x88\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "92: 6450021345187045785 0x 390795837174B123D 32", "\x02\x39\x32\xb2\x86\xfd\x98\xac\x8f\x8a\x83\xb3\x01\xfa\xc8\xd8\xf4\x02\xfa\xc8\xd8\xf4\x02\x40"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "93: 8586083292869710126 0x 170638545665B56030 78", "\x02\x39\x33\xdc\xc4\xd2\xb6\xf1\xd0\xf2\xa7\xee\x01\xe0\x80\xab\xdb\x0c\xe0\x80\xab\xdb\x0c\x9c\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "94: 88833738086841272 0x 90226635535C77DF3 82", "\x02\x39\x34\xf0\xee\xe3\xab\x8d\xf4\xcc\xbb\x02\xe6\xf7\xbb\xdc\x06\xe6\xf7\xbb\xdc\x06\xa4\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "95: 349662797203771308 0x 18558755186E9E69BE 118", "\x02\x39\x35\xd8\xbe\x80\xa8\xc4\x9d\xa0\xda\x09\xfc\xa6\xf3\xe9\x0d\xfc\xa6\xf3\xe9\x0d\xec\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "96: 163280929539230956 0x 40045642217DE7AE6 89", "\x02\x39\x36\xd8\xb3\xc0\xbc\xfd\xc8\x8b\xc4\x04\xcc\xeb\xf3\xfd\x02\xcc\xeb\xf3\xfd\x02\xb2\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "97: 4371241115470848126 0x 1128261220433FE664 112", "\x02\x39\x37\xfc\x81\xc7\xb3\xcf\xa8\xe2\xa9\x79\xc8\x99\xff\xb3\x08\xc8\x99\xff\xb3\x08\xe0\x01"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "98: 428374782176628108 0x 17127620106616AC9A 60", "\x02\x39\x38\x98\xb6\xfc\xa5\x88\xa6\xf2\xf1\x0b\xb4\xb2\xb5\xe1\x0c\xb4\xb2\xb5\xe1\x0c\x78"sv),
+TestCase("%s: %lld 0x%16u%08X %d", "99: 7023621621475593673 0x 1965201350BB6A8C7 58", "\x02\x39\x39\x92\xb7\xf2\x8c\xdd\xa9\xf5\xf8\xc2\x01\x8e\xa3\xb5\xbb\x01\x8e\xa3\xb5\xbb\x01\x74"sv),
+
+};
+
+} // namespace pw::test::tokenized_string_decoding
diff --git a/pw_tokenizer/pw_tokenizer_private/varint_decoding_test_data.h b/pw_tokenizer/pw_tokenizer_private/varint_decoding_test_data.h
new file mode 100644
index 0000000..8835f0c
--- /dev/null
+++ b/pw_tokenizer/pw_tokenizer_private/varint_decoding_test_data.h
@@ -0,0 +1,1012 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// AUTOGENERATED - DO NOT EDIT
+// This file contains test data generated by generate_decoding_test_data.cc.
+#pragma once
+
+#include <string_view>
+#include <tuple>
+
+namespace pw::test::varint_decoding {
+
+using namespace std::literals::string_view_literals;
+
+// clang-format off
+using TestCase = std::tuple<const char*, const char*, const char*, const char*, std::string_view>;
+
+inline constexpr TestCase kTestData[] = {
+
+// Important numbers
+TestCase("%d", "0", "%u", "0", "\x00"sv),
+TestCase("%d", "-32768", "%u", "4294934528", "\xff\xff\x03"sv),
+TestCase("%d", "-32767", "%u", "4294934529", "\xfd\xff\x03"sv),
+TestCase("%d", "32766", "%u", "32766", "\xfc\xff\x03"sv),
+TestCase("%d", "32767", "%u", "32767", "\xfe\xff\x03"sv),
+TestCase("%d", "-2147483648", "%u", "2147483648", "\xff\xff\xff\xff\x0f"sv),
+TestCase("%d", "-2147483647", "%u", "2147483649", "\xfd\xff\xff\xff\x0f"sv),
+TestCase("%d", "2147483646", "%u", "2147483646", "\xfc\xff\xff\xff\x0f"sv),
+TestCase("%d", "2147483647", "%u", "2147483647", "\xfe\xff\xff\xff\x0f"sv),
+TestCase("%lld", "-9223372036854775808", "%llu", "9223372036854775808", "\xff\xff\xff\xff\xff\xff\xff\xff\xff\x01"sv),
+TestCase("%lld", "-9223372036854775807", "%llu", "9223372036854775809", "\xfd\xff\xff\xff\xff\xff\xff\xff\xff\x01"sv),
+TestCase("%lld", "9223372036854775806", "%llu", "9223372036854775806", "\xfc\xff\xff\xff\xff\xff\xff\xff\xff\x01"sv),
+TestCase("%lld", "9223372036854775807", "%llu", "9223372036854775807", "\xfe\xff\xff\xff\xff\xff\xff\xff\xff\x01"sv),
+
+// Random 64-bit ints
+TestCase("%lld", "5922204476835468009", "%llu", "5922204476835468009", "\xd2\xcb\x8c\x90\x86\xe6\xf2\xaf\xa4\x01"sv),
+TestCase("%lld", "2004795154352895159", "%llu", "2004795154352895159", "\xee\xd2\x87\xea\xc5\xa4\xbb\xd2\x37"sv),
+TestCase("%lld", "7672492112153174982", "%llu", "7672492112153174982", "\x8c\x8f\x83\xee\x9c\xbb\x95\xfa\xd4\x01"sv),
+TestCase("%lld", "6325664365257058358", "%llu", "6325664365257058358", "\xec\xa0\xf3\xb7\xb6\x8e\xa3\xc9\xaf\x01"sv),
+TestCase("%lld", "4553661289274231220", "%llu", "4553661289274231220", "\xe8\xa6\x9a\xea\x9e\xb4\xed\xb1\x7e"sv),
+TestCase("%lld", "6372308406878241426", "%llu", "6372308406878241426", "\xa4\x8a\xb4\xf3\xfd\xae\xfe\xee\xb0\x01"sv),
+TestCase("%lld", "7156998241444634343", "%llu", "7156998241444634343", "\xce\xfb\xb8\xf6\xe5\xfe\xe1\xd2\xc6\x01"sv),
+TestCase("%lld", "1376699938710259787", "%llu", "1376699938710259787", "\x96\xa1\x84\x92\x9b\xd3\x82\x9b\x26"sv),
+TestCase("%lld", "3051600409971083011", "%llu", "3051600409971083011", "\x86\x9c\x9a\x8c\xf4\x99\xbb\xd9\x54"sv),
+TestCase("%lld", "6288685020493584850", "%llu", "6288685020493584850", "\xa4\xe7\xb8\xed\xa1\xed\xf2\xc5\xae\x01"sv),
+TestCase("%lld", "5705831195318701531", "%llu", "5705831195318701531", "\xb6\xb7\x82\x95\xb9\xcf\x97\xaf\x9e\x01"sv),
+TestCase("%lld", "2504359322455446492", "%llu", "2504359322455446492", "\xb8\xcf\xa8\xce\x9f\xe2\xa2\xc1\x45"sv),
+TestCase("%lld", "3679108774547190895", "%llu", "3679108774547190895", "\xde\xd1\xc3\xce\x81\xfc\xe8\x8e\x66"sv),
+TestCase("%lld", "1452704646622358274", "%llu", "1452704646622358274", "\x84\xac\xe1\x97\xbd\xcb\x85\xa9\x28"sv),
+TestCase("%lld", "1846464682573605487", "%llu", "1846464682573605487", "\xde\xa9\xd5\xf1\x90\xf6\xfa\x9f\x33"sv),
+TestCase("%lld", "4528166100111793966", "%llu", "4528166100111793966", "\xdc\xfc\x88\x92\xf5\xc4\xa3\xd7\x7d"sv),
+TestCase("%lld", "8393903718445878140", "%llu", "8393903718445878140", "\xf8\xad\xe0\x8c\xb1\xbd\x91\xfd\xe8\x01"sv),
+TestCase("%lld", "3957962835363152585", "%llu", "3957962835363152585", "\x92\x8b\xa6\xc0\xd0\x8e\xc1\xed\x6d"sv),
+TestCase("%lld", "3190545832108956470", "%llu", "3190545832108956470", "\xec\xec\xa8\xa7\xf6\xa1\x8c\xc7\x58"sv),
+TestCase("%lld", "5105279414768576647", "%llu", "5105279414768576647", "\x8e\xa2\xa9\xc6\x85\xa5\xcc\xd9\x8d\x01"sv),
+TestCase("%lld", "6049173436098818195", "%llu", "6049173436098818195", "\xa6\xc2\xb1\xb6\x96\xcc\xfd\xf2\xa7\x01"sv),
+TestCase("%lld", "3892265018717256260", "%llu", "3892265018717256260", "\x88\xe9\x91\xc5\xb2\x9a\x8d\x84\x6c"sv),
+TestCase("%lld", "6832059613091767623", "%llu", "6832059613091767623", "\x8e\x95\x85\xaa\xa6\x81\xad\xd0\xbd\x01"sv),
+TestCase("%lld", "810303956798710343", "%llu", "810303956798710343", "\x8e\x89\xa5\x91\xfa\xc9\xe3\xbe\x16"sv),
+TestCase("%lld", "970283311264054945", "%llu", "970283311264054945", "\xc2\xfa\x91\xb6\xfc\xe1\x91\xf7\x1a"sv),
+TestCase("%lld", "8832180626190956378", "%llu", "8832180626190956378", "\xb4\xdd\x97\x83\x82\xdf\x9a\x92\xf5\x01"sv),
+TestCase("%lld", "5816722312163363604", "%llu", "5816722312163363604", "\xa8\xac\xb0\xd9\xfc\x87\x93\xb9\xa1\x01"sv),
+TestCase("%lld", "4851344105826850048", "%llu", "4851344105826850048", "\x80\xd4\xf4\xef\xd7\xf0\xb7\xd3\x86\x01"sv),
+TestCase("%lld", "7829421709149921671", "%llu", "7829421709149921671", "\x8e\x86\xe7\xa9\xff\xe3\xd8\xa7\xd9\x01"sv),
+TestCase("%lld", "3885303859151835407", "%llu", "3885303859151835407", "\x9e\xc4\x98\x93\xca\xd1\xaf\xeb\x6b"sv),
+TestCase("%lld", "7185454812706950393", "%llu", "7185454812706950393", "\xf2\xb3\xa6\xd0\x9f\xc5\xee\xb7\xc7\x01"sv),
+TestCase("%lld", "4013414114257689954", "%llu", "4013414114257689954", "\xc4\xf5\xf2\x8d\xef\xb7\xc1\xb2\x6f"sv),
+TestCase("%lld", "964780727032512252", "%llu", "964780727032512252", "\xf8\xfb\xd3\x8e\xb5\xbd\xcb\xe3\x1a"sv),
+TestCase("%lld", "4207054084101944455", "%llu", "4207054084101944455", "\x8e\xd2\x85\x9c\xc9\xd9\xba\xe2\x74"sv),
+TestCase("%lld", "3970605724487453205", "%llu", "3970605724487453205", "\xaa\xa8\xa8\xf3\xd0\xb7\xb6\x9a\x6e"sv),
+TestCase("%lld", "7289505649862167307", "%llu", "7289505649862167307", "\x96\xac\xe6\x83\x8c\xb1\xc3\xa9\xca\x01"sv),
+TestCase("%lld", "1556249843733915123", "%llu", "1556249843733915123", "\xe6\x97\xa3\xd8\x99\xbf\xf4\x98\x2b"sv),
+TestCase("%lld", "646757550612212450", "%llu", "646757550612212450", "\xc4\xab\xd7\xc0\x99\xa4\xdf\xf9\x11"sv),
+TestCase("%lld", "551608669266414637", "%llu", "551608669266414637", "\xda\xb0\xb4\xaa\xb0\xca\xda\xa7\x0f"sv),
+TestCase("%lld", "5294527771240555016", "%llu", "5294527771240555016", "\x90\x88\xdf\xa3\x9a\xbd\xf8\xf9\x92\x01"sv),
+TestCase("%lld", "6427334826534330711", "%llu", "6427334826534330711", "\xae\x85\x9b\xc1\x94\xbe\xbd\xb2\xb2\x01"sv),
+TestCase("%lld", "4061389961487213535", "%llu", "4061389961487213535", "\xbe\x8f\xc3\xcc\xdb\xa9\xfa\xdc\x70"sv),
+TestCase("%lld", "6681126070454200740", "%llu", "6681126070454200740", "\xc8\xa6\xb1\x90\xea\xb0\x90\xb8\xb9\x01"sv),
+TestCase("%lld", "7083829078452288754", "%llu", "7083829078452288754", "\xe4\xf3\xd9\xeb\xfb\xc0\xe8\xce\xc4\x01"sv),
+TestCase("%lld", "4993075148853633222", "%llu", "4993075148853633222", "\x8c\x83\x83\x84\x97\xd9\xfb\xca\x8a\x01"sv),
+TestCase("%lld", "4002626457632111277", "%llu", "4002626457632111277", "\xda\xea\xef\xfb\xcd\xe3\x97\x8c\x6f"sv),
+TestCase("%lld", "6319581401334276901", "%llu", "6319581401334276901", "\xca\x8c\x8f\xbb\xa9\xf3\xd4\xb3\xaf\x01"sv),
+TestCase("%lld", "3705575452837642392", "%llu", "3705575452837642392", "\xb0\x82\xdc\xac\xb9\xcf\xec\xec\x66"sv),
+TestCase("%lld", "4291800171892412066", "%llu", "4291800171892412066", "\xc4\xba\xdc\xc9\x8e\xe1\xc4\x8f\x77"sv),
+TestCase("%lld", "6557245727662973594", "%llu", "6557245727662973594", "\xb4\x8a\xe5\xe4\xeb\x8f\x82\x80\xb6\x01"sv),
+TestCase("%lld", "3849893339411366329", "%llu", "3849893339411366329", "\xf2\x86\x8e\xec\x86\xe6\xc8\xed\x6a"sv),
+TestCase("%lld", "5454275499512896944", "%llu", "5454275499512896944", "\xe0\xf6\xa8\xf9\xe6\xaa\xbd\xb1\x97\x01"sv),
+TestCase("%lld", "5265826841850021261", "%llu", "5265826841850021261", "\x9a\x96\x80\xd3\x9e\xe7\xfc\x93\x92\x01"sv),
+TestCase("%lld", "9044083069173435164", "%llu", "9044083069173435164", "\xb8\xac\xa8\x93\xc8\xe8\x84\x83\xfb\x01"sv),
+TestCase("%lld", "7588739275664019217", "%llu", "7588739275664019217", "\xa2\xbc\x92\xb0\xc2\x8a\xcf\xd0\xd2\x01"sv),
+TestCase("%lld", "5076270483973408909", "%llu", "5076270483973408909", "\x9a\xa2\xc8\xd0\x84\xc7\xc4\xf2\x8c\x01"sv),
+TestCase("%lld", "4196804979465246477", "%llu", "4196804979465246477", "\x9a\xdc\xee\x89\x95\xf9\x85\xbe\x74"sv),
+TestCase("%lld", "6514086955327654755", "%llu", "6514086955327654755", "\xc6\xad\xfc\x84\xd4\xe4\xd7\xe6\xb4\x01"sv),
+TestCase("%lld", "9208944818170478756", "%llu", "9208944818170478756", "\xc8\x82\xce\xb3\xac\xa1\xdf\xcc\xff\x01"sv),
+TestCase("%lld", "4628058100229254151", "%llu", "4628058100229254151", "\x8e\xb0\xc5\x80\xcb\x94\x95\xba\x80\x01"sv),
+TestCase("%lld", "5505985599159795437", "%llu", "5505985599159795437", "\xda\xfb\xaf\x85\xe0\xae\x98\xe9\x98\x01"sv),
+TestCase("%lld", "1076793340331741575", "%llu", "1076793340331741575", "\x8e\xc6\xd6\xaf\xd0\xf5\xc4\xf1\x1d"sv),
+TestCase("%lld", "8835790874608329711", "%llu", "8835790874608329711", "\xde\x8f\xd7\xea\x90\xbf\x84\x9f\xf5\x01"sv),
+TestCase("%lld", "5264779051526567427", "%llu", "5264779051526567427", "\x86\xd8\xc5\xb2\xe8\xa9\xa0\x90\x92\x01"sv),
+TestCase("%lld", "3825505252128459194", "%llu", "3825505252128459194", "\xf4\xa6\x90\xf8\xc4\xb0\xf6\x96\x6a"sv),
+TestCase("%lld", "6197432947465793532", "%llu", "6197432947465793532", "\xf8\xbf\x81\x90\xc5\x9b\xda\x81\xac\x01"sv),
+TestCase("%lld", "6345326312201569781", "%llu", "6345326312201569781", "\xea\x97\xb6\xeb\xae\xaa\x90\x8f\xb0\x01"sv),
+TestCase("%lld", "1939578830432974807", "%llu", "1939578830432974807", "\xae\xdf\x99\x9d\xf0\xaa\xe2\xea\x35"sv),
+TestCase("%lld", "2091703771056304968", "%llu", "2091703771056304968", "\x90\xcd\xbf\x92\xeb\xdf\x9c\x87\x3a"sv),
+TestCase("%lld", "4252410814844357301", "%llu", "4252410814844357301", "\xea\xaa\x9e\x9d\xe3\xc6\xcc\x83\x76"sv),
+TestCase("%lld", "8660402439522563566", "%llu", "8660402439522563566", "\xdc\xd7\xd9\xba\xc9\x88\xf7\xaf\xf0\x01"sv),
+TestCase("%lld", "4095317955084962012", "%llu", "4095317955084962012", "\xb8\xa3\xf2\xb1\xee\xfe\xbe\xd5\x71"sv),
+TestCase("%lld", "3854481529221205087", "%llu", "3854481529221205087", "\xbe\xb1\xc6\xe8\xec\xa1\xef\xfd\x6a"sv),
+TestCase("%lld", "8433702361086030159", "%llu", "8433702361086030159", "\x9e\xc5\xbf\xde\xa3\xe7\xc3\x8a\xea\x01"sv),
+TestCase("%lld", "9155414173781516949", "%llu", "9155414173781516949", "\xaa\x8a\x97\xfd\xdf\xab\xc8\x8e\xfe\x01"sv),
+TestCase("%lld", "5127196966915280688", "%llu", "5127196966915280688", "\xe0\xbc\x90\xa7\xbe\x9e\xbb\xa7\x8e\x01"sv),
+TestCase("%lld", "4492265357832577542", "%llu", "4492265357832577542", "\x8c\x88\xbb\xa6\xd7\xe2\xdd\xd7\x7c"sv),
+TestCase("%lld", "8259453203008866922", "%llu", "8259453203008866922", "\xd4\xa9\xf2\xc4\xaf\xbb\xbc\x9f\xe5\x01"sv),
+TestCase("%lld", "5064367472631091289", "%llu", "5064367472631091289", "\xb2\xc1\xc0\xa3\xcb\xd8\x9f\xc8\x8c\x01"sv),
+TestCase("%lld", "8275937406817640763", "%llu", "8275937406817640763", "\xf6\x84\xdd\xcb\xa8\xce\x84\xda\xe5\x01"sv),
+TestCase("%lld", "1904310933731893847", "%llu", "1904310933731893847", "\xae\x89\xdb\xaf\x8d\xad\xbc\xed\x34"sv),
+TestCase("%lld", "997754361730893686", "%llu", "997754361730893686", "\xec\x9d\x88\x81\xc8\x93\xde\xd8\x1b"sv),
+TestCase("%lld", "4328168087819780921", "%llu", "4328168087819780921", "\xf2\x8c\xf0\xaa\xb7\xfc\xde\x90\x78"sv),
+TestCase("%lld", "256781249510239018", "%llu", "256781249510239018", "\xd4\xac\xcf\xa0\xf3\xcb\xa2\x90\x07"sv),
+TestCase("%lld", "7784993871819474513", "%llu", "7784993871819474513", "\xa2\x99\xa4\xc6\xc0\xab\xed\x89\xd8\x01"sv),
+TestCase("%lld", "2332871340892345868", "%llu", "2332871340892345868", "\x98\xd8\x9e\xc9\xfb\x87\x83\xe0\x40"sv),
+TestCase("%lld", "4894959346005630664", "%llu", "4894959346005630664", "\x90\xfb\xa6\x8a\xe6\xe5\xb1\xee\x87\x01"sv),
+TestCase("%lld", "1987972021100915124", "%llu", "1987972021100915124", "\xe8\xa6\x95\xd2\xa7\x81\xd9\x96\x37"sv),
+TestCase("%lld", "3114100465793448092", "%llu", "3114100465793448092", "\xb8\xe2\xf9\xa7\xfb\xf7\xc0\xb7\x56"sv),
+TestCase("%lld", "5784824274936856659", "%llu", "5784824274936856659", "\xa6\x81\xd8\xba\xeb\xc1\xe9\xc7\xa0\x01"sv),
+TestCase("%lld", "3207654208325833422", "%llu", "3207654208325833422", "\x9c\xcb\xe2\x98\xbe\xa0\xf0\x83\x59"sv),
+TestCase("%lld", "1164916386415315063", "%llu", "1164916386415315063", "\xee\xa1\x88\xd5\x81\xd2\xce\xaa\x20"sv),
+TestCase("%lld", "293049028994436992", "%llu", "293049028994436992", "\x80\xbe\xe4\xaf\xc1\xa2\x8f\x91\x08"sv),
+TestCase("%lld", "3140932642079254647", "%llu", "3140932642079254647", "\xee\xa1\x81\xa9\x97\xe6\xea\x96\x57"sv),
+TestCase("%lld", "1847278515897189565", "%llu", "1847278515897189565", "\xfa\x92\x9a\xd4\xbb\x81\xed\xa2\x33"sv),
+TestCase("%lld", "7940449326902609449", "%llu", "7940449326902609449", "\xd2\x98\xa8\xf0\xf5\xa5\x92\xb2\xdc\x01"sv),
+TestCase("%lld", "4324626723061920101", "%llu", "4324626723061920101", "\xca\xc5\xa3\xe4\xef\xc5\x94\x84\x78"sv),
+TestCase("%lld", "5240496025593009868", "%llu", "5240496025593009868", "\x98\xdb\x8a\xc5\xd6\xd7\xfd\xb9\x91\x01"sv),
+TestCase("%lld", "483285195300941883", "%llu", "483285195300941883", "\xf6\xc0\xf1\xf1\xe2\xd4\xfc\xb4\x0d"sv),
+TestCase("%lld", "5038839083535780307", "%llu", "5038839083535780307", "\xa6\xa7\x8a\xe8\xe2\xdc\xc6\xed\x8b\x01"sv),
+TestCase("%lld", "3649778670280906901", "%llu", "3649778670280906901", "\xaa\xe2\xa0\xee\x9f\x97\xcf\xa6\x65"sv),
+TestCase("%lld", "3630797309549363234", "%llu", "3630797309549363234", "\xc4\x80\x92\xf2\xd1\xba\x97\xe3\x64"sv),
+TestCase("%lld", "8062002663843236945", "%llu", "8062002663843236945", "\xa2\xb1\x88\xca\xab\xad\xfe\xe1\xdf\x01"sv),
+TestCase("%lld", "3467724524658884800", "%llu", "3467724524658884800", "\x80\x83\xa0\xf3\xa2\xc5\xea\x9f\x60"sv),
+TestCase("%lld", "5254956975956143854", "%llu", "5254956975956143854", "\xdc\xbb\x9d\xdd\xe2\xe1\xad\xed\x91\x01"sv),
+TestCase("%lld", "7712429266319135356", "%llu", "7712429266319135356", "\xf8\xc9\xba\xe4\xc6\xe3\x86\x88\xd6\x01"sv),
+TestCase("%lld", "5229125731492990833", "%llu", "5229125731492990833", "\xe2\xbd\xf4\xf7\xb9\x89\xcb\x91\x91\x01"sv),
+TestCase("%lld", "3291981441856444913", "%llu", "3291981441856444913", "\xe2\xe7\xc0\x90\xc0\xeb\xbb\xaf\x5b"sv),
+TestCase("%lld", "7968602228742132523", "%llu", "7968602228742132523", "\xd6\xcc\xfc\x88\xbe\xe0\x94\x96\xdd\x01"sv),
+TestCase("%lld", "4919847686728209916", "%llu", "4919847686728209916", "\xf8\xd7\xb2\xc7\xf2\xd9\xe7\xc6\x88\x01"sv),
+TestCase("%lld", "3071319170242969136", "%llu", "3071319170242969136", "\xe0\xf8\xef\xe2\xe8\xa0\xc2\x9f\x55"sv),
+TestCase("%lld", "6001314382948090489", "%llu", "6001314382948090489", "\xf2\x89\x82\xd1\xd1\xe8\xf9\xc8\xa6\x01"sv),
+TestCase("%lld", "5477548339997458345", "%llu", "5477548339997458345", "\xd2\xfe\xa1\xd9\xb4\xcc\x94\x84\x98\x01"sv),
+TestCase("%lld", "3119268858057174477", "%llu", "3119268858057174477", "\x9a\xb7\xfd\xb6\xfe\x9f\xef\xc9\x56"sv),
+TestCase("%lld", "2972175343159318480", "%llu", "2972175343159318480", "\xa0\xdf\xfc\xb2\xc6\xee\xa4\xbf\x52"sv),
+TestCase("%lld", "3141952960809904282", "%llu", "3141952960809904282", "\xb4\xa2\xfd\xa0\xc6\xe4\xba\x9a\x57"sv),
+TestCase("%lld", "7404358508754117308", "%llu", "7404358508754117308", "\xf8\x8a\xf1\xfe\xad\xb5\xc8\xc1\xcd\x01"sv),
+TestCase("%lld", "3962329461475475834", "%llu", "3962329461475475834", "\xf4\x85\xfc\xa4\xdc\xe9\x82\xfd\x6d"sv),
+TestCase("%lld", "8646072927884668865", "%llu", "8646072927884668865", "\x82\xbf\xff\x8a\x9c\xe1\x82\xfd\xef\x01"sv),
+TestCase("%lld", "6705183005166907218", "%llu", "6705183005166907218", "\xa4\xbd\xad\x8f\xdd\x9a\xcc\x8d\xba\x01"sv),
+TestCase("%lld", "8724568879186548965", "%llu", "8724568879186548965", "\xca\xe3\xbc\xe5\xf7\xca\xf2\x93\xf2\x01"sv),
+TestCase("%lld", "5275577307833715720", "%llu", "5275577307833715720", "\x90\xb0\xd2\xa8\x87\xe7\xce\xb6\x92\x01"sv),
+TestCase("%lld", "6278774444907314544", "%llu", "6278774444907314544", "\xe0\xf5\x81\xd8\xeb\x85\xd8\xa2\xae\x01"sv),
+TestCase("%lld", "4999233507689781945", "%llu", "4999233507689781945", "\xf2\xea\xa2\xf1\xeb\x98\xec\xe0\x8a\x01"sv),
+TestCase("%lld", "4257240410533938703", "%llu", "4257240410533938703", "\x9e\x98\x82\x95\x9f\xe6\xe0\x94\x76"sv),
+TestCase("%lld", "2788213596089450528", "%llu", "2788213596089450528", "\xc0\x80\x9c\x8d\xc7\xdf\xdc\xb1\x4d"sv),
+TestCase("%lld", "7141567845119971606", "%llu", "7141567845119971606", "\xac\xc4\xd2\xbb\xc2\x87\xf9\x9b\xc6\x01"sv),
+TestCase("%lld", "1507235571856201971", "%llu", "1507235571856201971", "\xe6\xe3\x92\xd5\x8a\xb1\xe3\xea\x29"sv),
+TestCase("%lld", "2967582888677845820", "%llu", "2967582888677845820", "\xf8\xac\xdd\x92\xc2\xba\xfc\xae\x52"sv),
+TestCase("%lld", "8120818212305491728", "%llu", "8120818212305491728", "\xa0\xdc\xee\xdf\xeb\xc8\xf8\xb2\xe1\x01"sv),
+TestCase("%lld", "8554950908976020523", "%llu", "8554950908976020523", "\xd6\xf0\xdd\x86\xd2\xa1\xa5\xb9\xed\x01"sv),
+TestCase("%lld", "850107129610567275", "%llu", "850107129610567275", "\xd6\xf9\xb7\x97\xc5\xfb\x97\xcc\x17"sv),
+TestCase("%lld", "3548380829014863866", "%llu", "3548380829014863866", "\xf4\xbf\xe5\x93\x8b\xe4\xb0\xbe\x62"sv),
+TestCase("%lld", "3619717899166040237", "%llu", "3619717899166040237", "\xda\x82\xd2\xee\x89\x90\xe9\xbb\x64"sv),
+TestCase("%lld", "25117409533303638", "%llu", "25117409533303638", "\xac\xed\xc8\x8e\xd4\x89\x9e\x59"sv),
+TestCase("%lld", "4766478454387430761", "%llu", "4766478454387430761", "\xd2\xd5\x86\xf5\xcf\xb9\xf7\xa5\x84\x01"sv),
+TestCase("%lld", "4030046377721652704", "%llu", "4030046377721652704", "\xc0\x97\x97\xc7\x82\xf5\xcc\xed\x6f"sv),
+TestCase("%lld", "7278057998974406311", "%llu", "7278057998974406311", "\xce\xba\xfa\xb8\x8e\xcc\xed\x80\xca\x01"sv),
+TestCase("%lld", "8751771536062737093", "%llu", "8751771536062737093", "\x8a\xab\xa7\xa6\xfb\xf5\xc4\xf4\xf2\x01"sv),
+TestCase("%lld", "3273441488341945355", "%llu", "3273441488341945355", "\x96\xb0\xae\x9a\x96\xec\xcc\xed\x5a"sv),
+TestCase("%lld", "5220278114967040707", "%llu", "5220278114967040707", "\x86\xcb\xd9\xf5\xb1\xd2\x93\xf2\x90\x01"sv),
+TestCase("%lld", "1352148195246416250", "%llu", "1352148195246416250", "\xf4\x85\xc9\xd5\xd3\xe7\xe5\xc3\x25"sv),
+TestCase("%lld", "5133499936442391273", "%llu", "5133499936442391273", "\xd2\xeb\x9c\xe2\xcc\xbf\xed\xbd\x8e\x01"sv),
+TestCase("%lld", "1676322249364352341", "%llu", "1676322249364352341", "\xaa\xb5\x91\xa7\x89\x8d\xbf\xc3\x2e"sv),
+TestCase("%lld", "5174306005887408729", "%llu", "5174306005887408729", "\xb2\xe9\xb6\x81\xba\xf9\xe9\xce\x8f\x01"sv),
+TestCase("%lld", "2333671424671038513", "%llu", "2333671424671038513", "\xe2\xf0\xf9\x9f\xfc\xf2\xee\xe2\x40"sv),
+TestCase("%lld", "6396005668587144321", "%llu", "6396005668587144321", "\x82\x92\xac\x98\x94\xd1\x96\xc3\xb1\x01"sv),
+TestCase("%lld", "4281900075145433102", "%llu", "4281900075145433102", "\x9c\xc0\x98\xec\xd1\xdb\xae\xec\x76"sv),
+TestCase("%lld", "8097189911969434437", "%llu", "8097189911969434437", "\x8a\xad\x84\xdd\xdf\xd4\xff\xde\xe0\x01"sv),
+TestCase("%lld", "1034989118514460270", "%llu", "1034989118514460270", "\xdc\xd9\x97\xb7\xd4\xc7\x82\xdd\x1c"sv),
+TestCase("%lld", "2879184457747193418", "%llu", "2879184457747193418", "\x94\xf9\xc6\xed\xcf\xc0\xf5\xf4\x4f"sv),
+TestCase("%lld", "2474224803315291647", "%llu", "2474224803315291647", "\xfe\xd7\x8c\x81\xb0\x96\x9b\xd6\x44"sv),
+TestCase("%lld", "3663174277129677254", "%llu", "3663174277129677254", "\x8c\xa7\xcd\xaa\x98\xe6\x9a\xd6\x65"sv),
+TestCase("%lld", "6086297047179889204", "%llu", "6086297047179889204", "\xe8\xa8\xd4\xea\xde\xba\xef\xf6\xa8\x01"sv),
+TestCase("%lld", "3403521624041286389", "%llu", "3403521624041286389", "\xea\xab\xbb\xa5\xcd\xb8\xde\xbb\x5e"sv),
+TestCase("%lld", "4129732865765686804", "%llu", "4129732865765686804", "\xa8\xf8\xc1\x93\xa8\x8a\xe1\xcf\x72"sv),
+TestCase("%lld", "8456228405596121943", "%llu", "8456228405596121943", "\xae\xad\xff\xf1\xd1\xbb\xc7\xda\xea\x01"sv),
+TestCase("%lld", "8297459921109698914", "%llu", "8297459921109698914", "\xc4\xa5\xc6\xfc\xc3\xf5\xbf\xa6\xe6\x01"sv),
+TestCase("%lld", "2758981765762549667", "%llu", "2758981765762549667", "\xc6\x9e\xf4\xa0\x8b\xd3\xef\xc9\x4c"sv),
+TestCase("%lld", "8848352886883606505", "%llu", "8848352886883606505", "\xd2\xff\xfe\xdf\xbd\x84\xd5\xcb\xf5\x01"sv),
+TestCase("%lld", "1195805645371054808", "%llu", "1195805645371054808", "\xb0\xeb\xfc\x8b\xc3\xb9\xad\x98\x21"sv),
+TestCase("%lld", "639367729021931858", "%llu", "639367729021931858", "\xa4\xc5\x9a\x93\xfc\xe3\xbe\xdf\x11"sv),
+TestCase("%lld", "2232000679497729109", "%llu", "2232000679497729109", "\xaa\x91\xa4\xfe\xda\xb2\xd4\xf9\x3d"sv),
+TestCase("%lld", "4168177859746898377", "%llu", "4168177859746898377", "\x92\x87\xe5\xad\xbe\xeb\xab\xd8\x73"sv),
+TestCase("%lld", "1260186573202255124", "%llu", "1260186573202255124", "\xa8\xb4\xff\xe6\xde\xc0\x8a\xfd\x22"sv),
+TestCase("%lld", "8933020158994285890", "%llu", "8933020158994285890", "\x84\x95\x90\xc4\xac\xa0\xbb\xf8\xf7\x01"sv),
+TestCase("%lld", "7418694076181766474", "%llu", "7418694076181766474", "\x94\x85\xdf\xc1\x9a\xbd\xbf\xf4\xcd\x01"sv),
+TestCase("%lld", "2904831533300168615", "%llu", "2904831533300168615", "\xce\xde\xad\xab\xf3\xb8\x84\xd0\x50"sv),
+TestCase("%lld", "7673041265242461202", "%llu", "7673041265242461202", "\xa4\xd0\x86\x9b\x92\x98\x8f\xfc\xd4\x01"sv),
+TestCase("%lld", "625118648746706096", "%llu", "625118648746706096", "\xe0\x92\xa8\x90\xab\x86\xef\xac\x11"sv),
+TestCase("%lld", "7640289034141395187", "%llu", "7640289034141395187", "\xe6\xa3\xfb\xd4\xb0\x99\xe1\x87\xd4\x01"sv),
+TestCase("%lld", "3317249430279903162", "%llu", "3317249430279903162", "\xf4\xde\x9a\xb3\x80\xb2\x9e\x89\x5c"sv),
+TestCase("%lld", "3172349440293538054", "%llu", "3172349440293538054", "\x8c\xa4\xd8\xc1\xc2\xc0\xb9\x86\x58"sv),
+TestCase("%lld", "2529759202900350655", "%llu", "2529759202900350655", "\xfe\xda\xb4\x82\xf0\xa5\xc1\x9b\x46"sv),
+TestCase("%lld", "7039166187898074754", "%llu", "7039166187898074754", "\x84\xda\xd7\x89\xca\x96\x92\xb0\xc3\x01"sv),
+TestCase("%lld", "4162847733300040370", "%llu", "4162847733300040370", "\xe4\xca\x9c\xd6\xa5\xfd\xb3\xc5\x73"sv),
+TestCase("%lld", "2754081621805980671", "%llu", "2754081621805980671", "\xfe\xef\xea\xe2\x95\xa9\xbb\xb8\x4c"sv),
+TestCase("%lld", "8048952107112912959", "%llu", "8048952107112912959", "\xfe\xe0\xc6\xe9\xfa\xd2\xcf\xb3\xdf\x01"sv),
+TestCase("%lld", "6063152674026149391", "%llu", "6063152674026149391", "\x9e\xd8\xce\xa4\xf5\xce\xd2\xa4\xa8\x01"sv),
+TestCase("%lld", "1860779983819210237", "%llu", "1860779983819210237", "\xfa\xf7\x9b\xcb\xaa\xe2\xe8\xd2\x33"sv),
+TestCase("%lld", "5403475014208127092", "%llu", "5403475014208127092", "\xe8\x81\x98\xe9\xb0\xf9\xff\xfc\x95\x01"sv),
+TestCase("%lld", "133586295505940516", "%llu", "133586295505940516", "\xc8\xc0\xef\xa7\xe5\x81\xcc\xda\x03"sv),
+TestCase("%lld", "5951209029639904448", "%llu", "5951209029639904448", "\x80\xf3\xc4\xbc\x9c\xc5\xf8\x96\xa5\x01"sv),
+TestCase("%lld", "476590412526583632", "%llu", "476590412526583632", "\xa0\xfd\x87\xa0\x90\x9d\x98\x9d\x0d"sv),
+TestCase("%lld", "6125277199963519765", "%llu", "6125277199963519765", "\xaa\xec\xff\xe9\xa0\xca\xad\x81\xaa\x01"sv),
+TestCase("%lld", "7182329075191030773", "%llu", "7182329075191030773", "\xea\x8f\xc5\xab\xb0\x8f\xe1\xac\xc7\x01"sv),
+TestCase("%lld", "3739073677431428339", "%llu", "3739073677431428339", "\xe6\xd3\x82\xc5\xdf\xec\xed\xe3\x67"sv),
+TestCase("%lld", "6781911602451840052", "%llu", "6781911602451840052", "\xe8\xf0\xef\xa3\xf3\xaa\x98\x9e\xbc\x01"sv),
+TestCase("%lld", "5097852782557067317", "%llu", "5097852782557067317", "\xea\xe0\xe7\xd8\x93\x86\x9b\xbf\x8d\x01"sv),
+TestCase("%lld", "2755779697069684711", "%llu", "2755779697069684711", "\xce\xdf\x93\xb1\x94\xc2\xbf\xbe\x4c"sv),
+TestCase("%lld", "4587140119625129692", "%llu", "4587140119625129692", "\xb8\xfb\x96\xd3\xd2\xe8\xe5\xa8\x7f"sv),
+TestCase("%lld", "7605101363071432517", "%llu", "7605101363071432517", "\x8a\xdd\xb1\xab\xad\xd9\xdf\x8a\xd3\x01"sv),
+TestCase("%lld", "785644280652604147", "%llu", "785644280652604147", "\xe6\x8b\xed\xc1\xf1\xd3\x95\xe7\x15"sv),
+TestCase("%lld", "8440954818933111077", "%llu", "8440954818933111077", "\xca\xe4\x89\xba\xf1\xeb\xa5\xa4\xea\x01"sv),
+TestCase("%lld", "3478923299848040400", "%llu", "3478923299848040400", "\xa0\xbf\xad\x84\xe4\x93\xcf\xc7\x60"sv),
+TestCase("%lld", "177390365453428882", "%llu", "177390365453428882", "\xa4\xa2\xed\xf6\x9e\xe6\x9b\xf6\x04"sv),
+TestCase("%lld", "2241679290640917987", "%llu", "2241679290640917987", "\xc6\x97\x83\xb0\x83\xdc\x85\x9c\x3e"sv),
+TestCase("%lld", "2690515256289614444", "%llu", "2690515256289614444", "\xd8\xe9\xca\x9e\xdc\xd7\xd0\xd6\x4a"sv),
+TestCase("%lld", "5284047940288483819", "%llu", "5284047940288483819", "\xd6\xc7\xbc\x90\xe1\xe6\xda\xd4\x92\x01"sv),
+TestCase("%lld", "7345613454774364586", "%llu", "7345613454774364586", "\xd4\xa6\xbf\xd2\x96\xa1\xee\xf0\xcb\x01"sv),
+TestCase("%lld", "7027616933390126935", "%llu", "7027616933390126935", "\xae\xbd\xe8\xc3\xbe\x97\x8e\x87\xc3\x01"sv),
+TestCase("%lld", "6370586723714568954", "%llu", "6370586723714568954", "\xf4\xcb\x9c\xdd\xea\xb7\xef\xe8\xb0\x01"sv),
+TestCase("%lld", "5815295050302678849", "%llu", "5815295050302678849", "\x82\x9d\x87\x96\xb3\x82\x8a\xb4\xa1\x01"sv),
+TestCase("%lld", "731720363738568228", "%llu", "731720363738568228", "\xc8\xd8\xf4\xb8\xe8\xf2\xcb\xa7\x14"sv),
+TestCase("%lld", "7895064303389261530", "%llu", "7895064303389261530", "\xb4\xfb\x91\xa6\xee\xc9\xf3\x90\xdb\x01"sv),
+TestCase("%lld", "9161537274294991160", "%llu", "9161537274294991160", "\xf0\xe4\xd3\xed\x8d\xe7\xa8\xa4\xfe\x01"sv),
+TestCase("%lld", "5626051873332343381", "%llu", "5626051873332343381", "\xaa\xc9\xa9\xde\xdc\x97\xe0\x93\x9c\x01"sv),
+TestCase("%lld", "1558715097695343451", "%llu", "1558715097695343451", "\xb6\xcd\xda\x82\xef\xc7\xd5\xa1\x2b"sv),
+TestCase("%lld", "418389641884993571", "%llu", "418389641884993571", "\xc6\xa0\x90\xe4\xb3\xca\xb5\xce\x0b"sv),
+TestCase("%lld", "366808135709449019", "%llu", "366808135709449019", "\xf6\xec\xfa\xda\xca\x83\x95\x97\x0a"sv),
+TestCase("%lld", "7559429297343079575", "%llu", "7559429297343079575", "\xae\x92\x85\xcd\x9e\xb9\xbe\xe8\xd1\x01"sv),
+TestCase("%lld", "891656539731055874", "%llu", "891656539731055874", "\x84\xd4\x96\x85\xc2\xb9\xe6\xdf\x18"sv),
+TestCase("%lld", "3187530423573642791", "%llu", "3187530423573642791", "\xce\xf8\xdb\xfd\x85\x82\xb1\xbc\x58"sv),
+TestCase("%lld", "3732114429930137548", "%llu", "3732114429930137548", "\x98\x9f\xc3\x91\x9d\x93\x91\xcb\x67"sv),
+TestCase("%lld", "4317425874089886561", "%llu", "4317425874089886561", "\xc2\x8d\xaa\x93\xa6\xfd\xc9\xea\x77"sv),
+TestCase("%lld", "8113837181842537067", "%llu", "8113837181842537067", "\xd6\xe9\xa5\xd1\xb1\xfb\x91\x9a\xe1\x01"sv),
+TestCase("%lld", "4647576880061632962", "%llu", "4647576880061632962", "\x84\xc7\xa2\xa6\x8d\xa3\xc1\xff\x80\x01"sv),
+TestCase("%lld", "2770024080011047550", "%llu", "2770024080011047550", "\xfc\xc9\xb1\xc5\xaf\x8e\x8d\xf1\x4c"sv),
+TestCase("%lld", "5715833275239865339", "%llu", "5715833275239865339", "\xf6\xef\xbc\xe0\x8f\x85\xdc\xd2\x9e\x01"sv),
+TestCase("%lld", "4697198541546045125", "%llu", "4697198541546045125", "\x8a\xfb\xad\xc5\xf9\xcb\xe6\xaf\x82\x01"sv),
+TestCase("%lld", "9140962295727161515", "%llu", "9140962295727161515", "\xd6\x82\xf4\xe0\xdd\xb1\x9c\xdb\xfd\x01"sv),
+TestCase("%lld", "1018100739973888361", "%llu", "1018100739973888361", "\xd2\xe5\xa5\xfb\xd2\xce\x82\xa1\x1c"sv),
+TestCase("%lld", "2450943428072886064", "%llu", "2450943428072886064", "\xe0\xbc\xe6\xad\xfd\x83\xc0\x83\x44"sv),
+TestCase("%lld", "4859753067646279323", "%llu", "4859753067646279323", "\xb6\xfa\xde\xd2\xd7\xea\xa7\xf1\x86\x01"sv),
+TestCase("%lld", "2904532547632894569", "%llu", "2904532547632894569", "\xd2\xc9\xc0\xc3\xd2\xbd\xfc\xce\x50"sv),
+TestCase("%lld", "549670438443165922", "%llu", "549670438443165922", "\xc4\xa3\x81\xaf\xbf\x96\xe9\xa0\x0f"sv),
+TestCase("%lld", "569815776002665926", "%llu", "569815776002665926", "\x8c\xb7\xeb\xc0\xbc\x9b\xb2\xe8\x0f"sv),
+TestCase("%lld", "3444356383434812300", "%llu", "3444356383434812300", "\x98\xae\xd9\xa7\xb7\xf8\xe7\xcc\x5f"sv),
+TestCase("%lld", "2151972984861854191", "%llu", "2151972984861854191", "\xde\xa7\xec\x8e\xe6\x81\xac\xdd\x3b"sv),
+TestCase("%lld", "2011546512680833085", "%llu", "2011546512680833085", "\xfa\xc0\xa2\x98\xa9\xb9\xb9\xea\x37"sv),
+TestCase("%lld", "4628394148228357534", "%llu", "4628394148228357534", "\xbc\xf6\xae\xda\x93\xfd\xad\xbb\x80\x01"sv),
+TestCase("%lld", "4692262078416922470", "%llu", "4692262078416922470", "\xcc\x9d\x9d\x91\xfd\xdf\xa1\x9e\x82\x01"sv),
+TestCase("%lld", "7858502056959645817", "%llu", "7858502056959645817", "\xf2\xe1\x8c\xbc\x82\xff\x80\x8f\xda\x01"sv),
+TestCase("%lld", "4919691286333696629", "%llu", "4919691286333696629", "\xea\xa9\x89\xe8\x98\xca\xa0\xc6\x88\x01"sv),
+TestCase("%lld", "5678520073874798535", "%llu", "5678520073874798535", "\x8e\xdf\xdb\xd3\xf9\xfa\x93\xce\x9d\x01"sv),
+TestCase("%lld", "1241746909443220722", "%llu", "1241746909443220722", "\xe4\x93\xeb\xe4\x85\x8f\xc9\xbb\x22"sv),
+TestCase("%lld", "5292137023426454572", "%llu", "5292137023426454572", "\xd8\x90\xb7\x80\xaf\xa5\xb9\xf1\x92\x01"sv),
+TestCase("%lld", "1941514050836526867", "%llu", "1941514050836526867", "\xa6\xdc\xd3\xe8\xc3\xaf\xd2\xf1\x35"sv),
+TestCase("%lld", "9145606885382902500", "%llu", "9145606885382902500", "\xc8\xcb\xea\xa9\xb7\xc0\xdc\xeb\xfd\x01"sv),
+TestCase("%lld", "3509957387132194637", "%llu", "3509957387132194637", "\x9a\xbd\xae\xcc\xb4\xe9\xef\xb5\x61"sv),
+TestCase("%lld", "8233948220616670992", "%llu", "8233948220616670992", "\xa0\xfc\x87\xcd\x80\x92\xee\xc4\xe4\x01"sv),
+TestCase("%lld", "8024983545482782384", "%llu", "8024983545482782384", "\xe0\x9a\xd9\xcd\x86\x81\xbc\xde\xde\x01"sv),
+TestCase("%lld", "6571000011597423770", "%llu", "6571000011597423770", "\xb4\x92\xd1\xe5\xc4\xec\xf0\xb0\xb6\x01"sv),
+TestCase("%lld", "5058360797369425151", "%llu", "5058360797369425151", "\xfe\xf3\xba\x8f\x89\x96\xf4\xb2\x8c\x01"sv),
+TestCase("%lld", "4606483085449113835", "%llu", "4606483085449113835", "\xd6\xc3\xd1\xc0\xb8\xfd\xc1\xed\x7f"sv),
+TestCase("%lld", "6529118911438932678", "%llu", "6529118911438932678", "\x8c\xab\xd2\xc9\xd4\xc3\x8b\x9c\xb5\x01"sv),
+TestCase("%lld", "4203154114144238442", "%llu", "4203154114144238442", "\xd4\xed\xcf\xdc\xb8\x99\xcd\xd4\x74"sv),
+TestCase("%lld", "4463425652718897265", "%llu", "4463425652718897265", "\xe2\xa1\xe8\xfa\xf3\xfe\xa2\xf1\x7b"sv),
+TestCase("%lld", "75152304545876858", "%llu", "75152304545876858", "\xf4\xad\x92\x88\xee\xa7\xff\x8a\x02"sv),
+TestCase("%lld", "5434900075521908661", "%llu", "5434900075521908661", "\xea\xfe\xd0\xf3\xd8\xb4\xd2\xec\x96\x01"sv),
+TestCase("%lld", "5308383100912368033", "%llu", "5308383100912368033", "\xc2\xe6\x8b\xd9\xc4\x93\x95\xab\x93\x01"sv),
+TestCase("%lld", "715267470600062067", "%llu", "715267470600062067", "\xe6\x91\x92\x88\xb2\xfe\x91\xed\x13"sv),
+TestCase("%lld", "5171700275229723650", "%llu", "5171700275229723650", "\x84\xb0\xdc\xdd\xfb\xff\xc8\xc5\x8f\x01"sv),
+TestCase("%lld", "3926011308434788247", "%llu", "3926011308434788247", "\xae\xbe\xe0\xf6\xfd\x9e\xff\xfb\x6c"sv),
+TestCase("%lld", "5440336360732713747", "%llu", "5440336360732713747", "\xa6\x8c\xeb\x81\x92\xc6\xfa\xff\x96\x01"sv),
+TestCase("%lld", "1478056893002169192", "%llu", "1478056893002169192", "\xd0\xed\xb1\x93\xb8\xba\x8e\x83\x29"sv),
+TestCase("%lld", "3160377536621687761", "%llu", "3160377536621687761", "\xa2\xcf\xad\xa7\x81\xa8\xf5\xdb\x57"sv),
+TestCase("%lld", "7128491551588143718", "%llu", "7128491551588143718", "\xcc\xc9\xc6\xcc\x87\xd3\xbe\xed\xc5\x01"sv),
+TestCase("%lld", "7376169372313052441", "%llu", "7376169372313052441", "\xb2\x94\xaf\xfd\xd4\xbd\xb5\xdd\xcc\x01"sv),
+TestCase("%lld", "193231861624509923", "%llu", "193231861624509923", "\xc6\xd7\xd4\xf6\xd6\xd6\xbf\xae\x05"sv),
+TestCase("%lld", "1955429981000213095", "%llu", "1955429981000213095", "\xce\x89\xd5\x8a\xa3\xcd\x8a\xa3\x36"sv),
+TestCase("%lld", "4888125773563526975", "%llu", "4888125773563526975", "\xfe\xfc\xfb\xea\xc3\x9f\x8e\xd6\x87\x01"sv),
+TestCase("%lld", "5921662978695331131", "%llu", "5921662978695331131", "\xf6\xf4\xbb\xcb\xda\xc6\xfc\xad\xa4\x01"sv),
+TestCase("%lld", "7673779374634366645", "%llu", "7673779374634366645", "\xea\xda\xe7\xdc\xe1\xeb\xde\xfe\xd4\x01"sv),
+TestCase("%lld", "48609148367913892", "%llu", "48609148367913892", "\xc8\x8e\xee\xd4\xe9\xf0\xd8\xac\x01"sv),
+TestCase("%lld", "5854341954344571365", "%llu", "5854341954344571365", "\xca\xc7\xfe\xe5\xac\xbf\xe6\xbe\xa2\x01"sv),
+TestCase("%lld", "4673618806014082273", "%llu", "4673618806014082273", "\xc2\xf3\xef\xc0\xd9\xe2\x83\xdc\x81\x01"sv),
+TestCase("%lld", "2326184191199164007", "%llu", "2326184191199164007", "\xce\xd9\xf1\xe8\xcf\x8c\xa2\xc8\x40"sv),
+TestCase("%lld", "3495015264127128735", "%llu", "3495015264127128735", "\xbe\xd2\xec\xed\xb0\xf7\xe4\x80\x61"sv),
+TestCase("%lld", "1424330851786829530", "%llu", "1424330851786829530", "\xb4\xbb\x91\xf6\x9f\xd7\x9e\xc4\x27"sv),
+TestCase("%lld", "5191701707746974065", "%llu", "5191701707746974065", "\xe2\xe5\xe0\xe0\xc8\xcc\xd0\x8c\x90\x01"sv),
+TestCase("%lld", "6389735809259880362", "%llu", "6389735809259880362", "\xd4\xfe\xb9\xf8\xa7\xb7\xf3\xac\xb1\x01"sv),
+TestCase("%lld", "7792983928750025393", "%llu", "7792983928750025393", "\xe2\xaa\xc9\xd7\x83\xe6\x9e\xa6\xd8\x01"sv),
+TestCase("%lld", "4564825427452770815", "%llu", "4564825427452770815", "\xfe\xe7\xf1\x8f\xce\xa2\xc2\xd9\x7e"sv),
+TestCase("%lld", "943049921933903574", "%llu", "943049921933903574", "\xac\x9b\xa4\xa5\x8a\xba\xb1\x96\x1a"sv),
+TestCase("%lld", "1621021906259841134", "%llu", "1621021906259841134", "\xdc\xc1\xf4\xfb\xb9\xb5\x83\xff\x2c"sv),
+TestCase("%lld", "7630179381739069820", "%llu", "7630179381739069820", "\xf8\x85\x89\xd1\x94\xee\xeb\xe3\xd3\x01"sv),
+TestCase("%lld", "8025284571884195522", "%llu", "8025284571884195522", "\x84\xbb\xa4\x89\x8c\xf3\xc4\xdf\xde\x01"sv),
+TestCase("%lld", "5839395391962199967", "%llu", "5839395391962199967", "\xbe\xde\xad\x8f\xf5\xca\xd9\x89\xa2\x01"sv),
+TestCase("%lld", "1310759272316304459", "%llu", "1310759272316304459", "\x96\xf1\xb3\xc7\xa1\xa7\xe0\xb0\x24"sv),
+TestCase("%lld", "3663531610526815443", "%llu", "3663531610526815443", "\xa6\xd3\xec\xcd\xdd\xa5\xbd\xd7\x65"sv),
+TestCase("%lld", "3661335273992329249", "%llu", "3661335273992329249", "\xc2\xe0\xdc\x9e\x8f\xc2\xd6\xcf\x65"sv),
+TestCase("%lld", "3545512604455022758", "%llu", "3545512604455022758", "\xcc\xc2\xb1\xf3\xb8\xbb\x98\xb4\x62"sv),
+TestCase("%lld", "4908269103466540013", "%llu", "4908269103466540013", "\xda\xaf\xaf\xe2\xd2\xaf\xd6\x9d\x88\x01"sv),
+TestCase("%lld", "9012690640501821443", "%llu", "9012690640501821443", "\x86\xc0\xca\x99\xdc\x98\xc1\x93\xfa\x01"sv),
+TestCase("%lld", "6541851858139706195", "%llu", "6541851858139706195", "\xa6\xcd\xab\x9a\xda\xe6\xa9\xc9\xb5\x01"sv),
+TestCase("%lld", "6391701285149964727", "%llu", "6391701285149964727", "\xee\xe6\xab\xbc\x88\x9d\xf1\xb3\xb1\x01"sv),
+TestCase("%lld", "4510904210489015127", "%llu", "4510904210489015127", "\xae\xed\x97\xa0\xd9\xde\xf9\x99\x7d"sv),
+TestCase("%lld", "5119199357584793664", "%llu", "5119199357584793664", "\x80\x81\xff\xb3\xad\xac\x86\x8b\x8e\x01"sv),
+TestCase("%lld", "1751202991906984173", "%llu", "1751202991906984173", "\xda\xc3\xbc\xe3\xf8\xf5\xc2\xcd\x30"sv),
+TestCase("%lld", "4471970899266236465", "%llu", "4471970899266236465", "\xe2\xc0\xaf\x8a\xdc\xf5\xd0\x8f\x7c"sv),
+TestCase("%lld", "7912743439818658557", "%llu", "7912743439818658557", "\xfa\xcb\x8c\x93\x85\x8f\xdb\xcf\xdb\x01"sv),
+TestCase("%lld", "6642713590355959471", "%llu", "6642713590355959471", "\xde\xfa\x9c\x9d\x9b\xb4\xd4\xaf\xb8\x01"sv),
+TestCase("%lld", "7865046769076850396", "%llu", "7865046769076850396", "\xb8\xbb\xc6\xca\xb3\x97\xa1\xa6\xda\x01"sv),
+TestCase("%lld", "4109397703148639581", "%llu", "4109397703148639581", "\xba\xd5\xd8\x8f\x88\xdc\xc1\x87\x72"sv),
+TestCase("%lld", "436719658465345699", "%llu", "436719658465345699", "\xc6\x82\x8e\xd4\xe5\x8d\xc5\x8f\x0c"sv),
+TestCase("%lld", "2553774266074455395", "%llu", "2553774266074455395", "\xc6\xf5\xb6\xde\xc3\x8a\xea\xf0\x46"sv),
+TestCase("%lld", "6212224561138197241", "%llu", "6212224561138197241", "\xf2\x8b\xcb\xf9\xe2\xd4\xa0\xb6\xac\x01"sv),
+TestCase("%lld", "53039180701654763", "%llu", "53039180701654763", "\xd6\xfb\xb2\xce\xd2\xb6\xb7\xbc\x01"sv),
+TestCase("%lld", "5419660422277913790", "%llu", "5419660422277913790", "\xfc\xf2\xf2\x91\x90\x9c\xc0\xb6\x96\x01"sv),
+TestCase("%lld", "7637231844465361135", "%llu", "7637231844465361135", "\xde\x83\xc6\xca\xc2\xf9\xf2\xfc\xd3\x01"sv),
+TestCase("%lld", "1955771002711010609", "%llu", "1955771002711010609", "\xe2\xb4\xcb\xe9\xac\xd7\xa5\xa4\x36"sv),
+TestCase("%lld", "7552146003088329848", "%llu", "7552146003088329848", "\xf0\xb1\xba\xdf\xdb\xb1\xce\xce\xd1\x01"sv),
+TestCase("%lld", "4550116504720438569", "%llu", "4550116504720438569", "\xd2\xd4\x87\xe8\xcf\xb6\xa1\xa5\x7e"sv),
+TestCase("%lld", "757246289038783544", "%llu", "757246289038783544", "\xf0\xd0\x9d\x9a\x9c\xdf\xa3\x82\x15"sv),
+TestCase("%lld", "4684454901623556905", "%llu", "4684454901623556905", "\xd2\xac\xe0\x99\xbd\xba\xc3\x82\x82\x01"sv),
+TestCase("%lld", "2100956932097049429", "%llu", "2100956932097049429", "\xaa\x8d\xd0\xa0\xd9\xcc\x8c\xa8\x3a"sv),
+TestCase("%lld", "6234369280162760629", "%llu", "6234369280162760629", "\xea\x9e\xac\xeb\x88\xf5\xf6\x84\xad\x01"sv),
+TestCase("%lld", "1690133901535631706", "%llu", "1690133901535631706", "\xb4\xe5\x82\xfe\x84\xf5\xc7\xf4\x2e"sv),
+TestCase("%lld", "8944940508995398022", "%llu", "8944940508995398022", "\x8c\xa6\xe1\xfa\x84\x80\xe8\xa2\xf8\x01"sv),
+TestCase("%lld", "5911305799563392167", "%llu", "5911305799563392167", "\xce\xd2\xf5\xda\xc4\xd3\x96\x89\xa4\x01"sv),
+TestCase("%lld", "4077388965675931531", "%llu", "4077388965675931531", "\x96\xde\xb0\xe3\xa9\xea\xe5\x95\x71"sv),
+TestCase("%lld", "1245196532623960857", "%llu", "1245196532623960857", "\xb2\xac\xcb\xa9\xc5\xe9\xe9\xc7\x22"sv),
+TestCase("%lld", "8856210144434134086", "%llu", "8856210144434134086", "\x8c\xf1\xa8\xb3\x88\x8d\xca\xe7\xf5\x01"sv),
+TestCase("%lld", "8547254512181346310", "%llu", "8547254512181346310", "\x8c\x80\xe4\xb6\xb1\xac\xf9\x9d\xed\x01"sv),
+TestCase("%lld", "6269182192420305924", "%llu", "6269182192420305924", "\x88\xf0\xd5\xb6\xa1\xff\xcd\x80\xae\x01"sv),
+TestCase("%lld", "6831000756220180063", "%llu", "6831000756220180063", "\xbe\xd9\xed\xee\xdb\xbf\xcb\xcc\xbd\x01"sv),
+TestCase("%lld", "3168030316008523010", "%llu", "3168030316008523010", "\x84\x94\xda\xf5\xb2\xb2\x8d\xf7\x57"sv),
+TestCase("%lld", "2407920475746907080", "%llu", "2407920475746907080", "\x90\xcf\xc5\xc4\xc7\xba\xd3\xea\x42"sv),
+TestCase("%lld", "7828361129715530032", "%llu", "7828361129715530032", "\xe0\xe4\xdb\xe2\x92\xbe\xf6\xa3\xd9\x01"sv),
+TestCase("%lld", "5162541696057507903", "%llu", "5162541696057507903", "\xfe\x80\xb4\xba\xbf\x94\x84\xa5\x8f\x01"sv),
+TestCase("%lld", "4684430121285811332", "%llu", "4684430121285811332", "\x88\xf2\x8b\xa6\x89\x98\xb8\x82\x82\x01"sv),
+TestCase("%lld", "7069327273055818092", "%llu", "7069327273055818092", "\xd8\xc5\xc8\xf7\xe5\xec\xa5\x9b\xc4\x01"sv),
+TestCase("%lld", "1094908319066476988", "%llu", "1094908319066476988", "\xf8\xc6\x9e\xb9\x95\xd4\xf2\xb1\x1e"sv),
+TestCase("%lld", "8623611379126599086", "%llu", "8623611379126599086", "\xdc\xf6\xdb\xdd\x80\xb7\x9c\xad\xef\x01"sv),
+TestCase("%lld", "4188361875415362989", "%llu", "4188361875415362989", "\xda\xb6\xdf\xee\xe9\xbb\x86\xa0\x74"sv),
+TestCase("%lld", "8306128555371214226", "%llu", "8306128555371214226", "\xa4\xd6\x89\x9a\xba\xfa\xa5\xc5\xe6\x01"sv),
+TestCase("%lld", "6350883509347762576", "%llu", "6350883509347762576", "\xa0\x86\xfa\x88\xe8\xb9\xef\xa2\xb0\x01"sv),
+TestCase("%lld", "6330969482079692999", "%llu", "6330969482079692999", "\x8e\xa3\x97\xf3\xee\xcc\x8f\xdc\xaf\x01"sv),
+TestCase("%lld", "3843711385867305612", "%llu", "3843711385867305612", "\x98\x8a\xf6\xdb\xff\xc8\xcd\xd7\x6a"sv),
+TestCase("%lld", "7659651832434418091", "%llu", "7659651832434418091", "\xd6\x86\xe3\xf6\xc9\xb0\xc6\xcc\xd4\x01"sv),
+TestCase("%lld", "5535789636778214509", "%llu", "5535789636778214509", "\xda\x91\xa2\xb7\x88\xd6\x89\xd3\x99\x01"sv),
+TestCase("%lld", "5087759841087488516", "%llu", "5087759841087488516", "\x88\x88\xa4\xe8\xd1\xa7\xad\x9b\x8d\x01"sv),
+TestCase("%lld", "8891437578007474517", "%llu", "8891437578007474517", "\xaa\xc5\x8f\xbb\xc9\xd7\xdd\xe4\xf6\x01"sv),
+TestCase("%lld", "728151566226426022", "%llu", "728151566226426022", "\xcc\xa2\x90\xf9\xba\xff\xf4\x9a\x14"sv),
+TestCase("%lld", "2483479174720276235", "%llu", "2483479174720276235", "\x96\x9c\xc2\x84\xd8\xc9\x8b\xf7\x44"sv),
+TestCase("%lld", "6359340322607236594", "%llu", "6359340322607236594", "\xe4\x97\x94\xed\x90\x95\xf5\xc0\xb0\x01"sv),
+TestCase("%lld", "6416832625845083020", "%llu", "6416832625845083020", "\x98\xee\x90\xea\xcf\xd1\x95\x8d\xb2\x01"sv),
+TestCase("%lld", "5711184686518624115", "%llu", "5711184686518624115", "\xe6\xfd\xbe\xe9\xd2\x8d\x9a\xc2\x9e\x01"sv),
+TestCase("%lld", "3612059979445064775", "%llu", "3612059979445064775", "\x8e\xd1\xd2\xe3\xbd\xda\xce\xa0\x64"sv),
+TestCase("%lld", "8992370046494348158", "%llu", "8992370046494348158", "\xfc\xcd\xc4\xe2\xbc\xba\xa8\xcb\xf9\x01"sv),
+TestCase("%lld", "8599514281591661754", "%llu", "8599514281591661754", "\xf4\x92\xdb\xb2\xa9\xab\xce\xd7\xee\x01"sv),
+TestCase("%lld", "63057463895113152", "%llu", "63057463895113152", "\x80\xf7\xbd\xfe\xbc\x9b\x83\xe0\x01"sv),
+TestCase("%lld", "1367741239369008397", "%llu", "1367741239369008397", "\x9a\x94\xf0\xd3\xa3\xda\x98\xfb\x25"sv),
+TestCase("%lld", "22816942625156124", "%llu", "22816942625156124", "\xb8\xb0\xc3\x80\xee\xf8\x87\x51"sv),
+TestCase("%lld", "5473904087316166353", "%llu", "5473904087316166353", "\xa2\xdb\xcf\xb0\xfe\xb0\x9b\xf7\x97\x01"sv),
+TestCase("%lld", "2176574837538100650", "%llu", "2176574837538100650", "\xd4\xd6\x98\xa8\x8c\xd2\xdf\xb4\x3c"sv),
+TestCase("%lld", "1822527202856628083", "%llu", "1822527202856628083", "\xe6\xbd\x8b\xfc\xb6\xb5\xf5\xca\x32"sv),
+TestCase("%lld", "6704455477950302825", "%llu", "6704455477950302825", "\xd2\xa9\xb2\xfa\x88\xaf\x81\x8b\xba\x01"sv),
+TestCase("%lld", "3032341809633898873", "%llu", "3032341809633898873", "\xf2\xc5\xd9\x8c\xea\xb3\x85\x95\x54"sv),
+TestCase("%lld", "4392190674790178652", "%llu", "4392190674790178652", "\xb8\xfd\xb6\xca\xbb\x89\x99\xf4\x79"sv),
+TestCase("%lld", "6856757614633255152", "%llu", "6856757614633255152", "\xe0\x83\xdc\xb2\x99\xae\x8c\xa8\xbe\x01"sv),
+TestCase("%lld", "353529678637100035", "%llu", "353529678637100035", "\x86\xb0\xc7\x8b\xd4\xd7\xfe\xe7\x09"sv),
+TestCase("%lld", "3224702228554841730", "%llu", "3224702228554841730", "\x84\xfa\xdf\xf6\xee\xe5\xb8\xc0\x59"sv),
+TestCase("%lld", "8122609781749075974", "%llu", "8122609781749075974", "\x8c\xc0\xa2\xa4\xf4\xa3\xa7\xb9\xe1\x01"sv),
+TestCase("%lld", "3988658283410987938", "%llu", "3988658283410987938", "\xc4\xee\x8c\x92\xee\xe4\xc7\xda\x6e"sv),
+TestCase("%lld", "7588966234436262844", "%llu", "7588966234436262844", "\xf8\xee\x8e\x80\xa1\xa5\xb6\xd1\xd2\x01"sv),
+TestCase("%lld", "72920210115635164", "%llu", "72920210115635164", "\xb8\x9f\x9d\xce\xee\xa2\x88\x83\x02"sv),
+TestCase("%lld", "8168752409782378632", "%llu", "8168752409782378632", "\x90\x82\x8f\xb3\xad\xc2\x9e\xdd\xe2\x01"sv),
+TestCase("%lld", "4655403017017936000", "%llu", "4655403017017936000", "\x80\xa2\x96\xba\x9d\x98\xa8\x9b\x81\x01"sv),
+TestCase("%lld", "8183602528750449920", "%llu", "8183602528750449920", "\x80\xd4\xc5\xd2\x85\xc9\xff\x91\xe3\x01"sv),
+TestCase("%lld", "288952216557681443", "%llu", "288952216557681443", "\xc6\xec\xdb\x8f\xd2\xa0\xc8\x82\x08"sv),
+TestCase("%lld", "1676746537447019293", "%llu", "1676746537447019293", "\xba\xcc\xaa\xee\xf1\x85\x80\xc5\x2e"sv),
+TestCase("%lld", "3219359012813218523", "%llu", "3219359012813218523", "\xb6\x8b\xf9\xb2\xe3\xfd\xba\xad\x59"sv),
+TestCase("%lld", "9123809425872405130", "%llu", "9123809425872405130", "\x94\x8a\xe2\xb7\xa6\x95\xa4\x9e\xfd\x01"sv),
+TestCase("%lld", "7496923004744506260", "%llu", "7496923004744506260", "\xa8\xee\x80\xe4\x93\xf0\xb5\x8a\xd0\x01"sv),
+TestCase("%lld", "2810327810830374172", "%llu", "2810327810830374172", "\xb8\xd4\x90\xaf\xa2\x90\xa5\x80\x4e"sv),
+TestCase("%lld", "5886875793686089828", "%llu", "5886875793686089828", "\xc8\xc1\x8a\xf8\x84\x96\xb1\xb2\xa3\x01"sv),
+TestCase("%lld", "1043235540762466020", "%llu", "1043235540762466020", "\xc8\xdb\xac\x83\xce\xcc\xa8\xfa\x1c"sv),
+TestCase("%lld", "8352418169916257811", "%llu", "8352418169916257811", "\xa6\xd8\x83\x96\xd2\x84\xe0\xe9\xe7\x01"sv),
+TestCase("%lld", "674709631558538790", "%llu", "674709631558538790", "\xcc\xd8\xa6\xb9\xb9\xb5\x86\xdd\x12"sv),
+TestCase("%lld", "6425607598961991251", "%llu", "6425607598961991251", "\xa6\xf9\xf0\xa2\xa4\x84\xac\xac\xb2\x01"sv),
+TestCase("%lld", "6117996971473322038", "%llu", "6117996971473322038", "\xec\x80\xaa\xd9\x97\xf5\xbe\xe7\xa9\x01"sv),
+TestCase("%lld", "2394571431170796410", "%llu", "2394571431170796410", "\xf4\xad\xdf\x83\xf3\x81\x9d\xbb\x42"sv),
+TestCase("%lld", "9192320418779887179", "%llu", "9192320418779887179", "\x96\xd9\x8f\xef\xf8\xad\xd7\x91\xff\x01"sv),
+TestCase("%lld", "891164515808164858", "%llu", "891164515808164858", "\xf4\xef\xed\x92\xfa\xd9\x86\xde\x18"sv),
+TestCase("%lld", "4569690489566792107", "%llu", "4569690489566792107", "\xd6\xa6\xe0\xc1\xbf\xd2\xe6\xea\x7e"sv),
+TestCase("%lld", "4245075167591901055", "%llu", "4245075167591901055", "\xfe\x9d\xad\x92\xf4\xd7\xc4\xe9\x75"sv),
+TestCase("%lld", "9019812505721196330", "%llu", "9019812505721196330", "\xd4\xbc\x9a\x82\xeb\xeb\xe7\xac\xfa\x01"sv),
+TestCase("%lld", "979687222629424543", "%llu", "979687222629424543", "\xbe\xf6\x86\xfd\xd3\x95\xc6\x98\x1b"sv),
+TestCase("%lld", "4114720885129059820", "%llu", "4114720885129059820", "\xd8\xf7\xfc\xbf\x84\xb6\xb6\x9a\x72"sv),
+TestCase("%lld", "2602113737417537288", "%llu", "2602113737417537288", "\x90\xdc\xfd\xfe\x8a\xaa\xc8\x9c\x48"sv),
+TestCase("%lld", "4025468271985009702", "%llu", "4025468271985009702", "\xcc\x90\xad\xe4\x98\x84\xab\xdd\x6f"sv),
+TestCase("%lld", "3036291044299697402", "%llu", "3036291044299697402", "\xf4\xb3\xf7\xa2\xc5\xa7\x89\xa3\x54"sv),
+TestCase("%lld", "1070534622568918680", "%llu", "1070534622568918680", "\xb0\xba\xd1\x9f\xa7\xe4\xa6\xdb\x1d"sv),
+TestCase("%lld", "5631405388638357750", "%llu", "5631405388638357750", "\xec\xa3\x94\x80\xaa\xd7\xe2\xa6\x9c\x01"sv),
+TestCase("%lld", "53610470084017162", "%llu", "53610470084017162", "\x94\xf0\xcb\x99\x88\x9c\xbb\xbe\x01"sv),
+TestCase("%lld", "6477169593541844553", "%llu", "6477169593541844553", "\x92\xf9\xae\xd0\xb0\xdb\xc3\xe3\xb3\x01"sv),
+TestCase("%lld", "1775599305804553124", "%llu", "1775599305804553124", "\xc8\xee\xc8\xde\xa7\x8a\x99\xa4\x31"sv),
+TestCase("%lld", "720129488086281194", "%llu", "720129488086281194", "\xd4\xaf\xd5\x9a\x87\xfd\xb4\xfe\x13"sv),
+TestCase("%lld", "6711270492673007555", "%llu", "6711270492673007555", "\x86\xff\xae\xb5\x91\xbd\x9c\xa3\xba\x01"sv),
+TestCase("%lld", "7513165068981918756", "%llu", "7513165068981918756", "\xc8\xf0\xd2\xb8\xdc\xf4\x8f\xc4\xd0\x01"sv),
+TestCase("%lld", "2983712669290254370", "%llu", "2983712669290254370", "\xc4\xc0\x86\xce\xa8\xb7\xa3\xe8\x52"sv),
+TestCase("%lld", "3737216259576487553", "%llu", "3737216259576487553", "\x82\xfa\xf6\x8d\xe6\x98\xa1\xdd\x67"sv),
+TestCase("%lld", "2274224013798231728", "%llu", "2274224013798231728", "\xe0\xba\xda\xa1\x9e\xac\xd5\x8f\x3f"sv),
+TestCase("%lld", "3565198557459535882", "%llu", "3565198557459535882", "\x94\xe0\xe2\xb1\xdc\xcc\x90\xfa\x62"sv),
+TestCase("%lld", "3732880420468774142", "%llu", "3732880420468774142", "\xfc\xd3\xea\xb0\xdf\xbd\xed\xcd\x67"sv),
+TestCase("%lld", "5371965991102827603", "%llu", "5371965991102827603", "\xa6\xb1\xce\xd2\xec\xa6\x87\x8d\x95\x01"sv),
+TestCase("%lld", "484335498791767840", "%llu", "484335498791767840", "\xc0\xbc\xc2\xd9\xbd\xa4\xda\xb8\x0d"sv),
+TestCase("%lld", "5776656679787538668", "%llu", "5776656679787538668", "\xd8\xa3\xfd\xc1\x9d\xa9\xe7\xaa\xa0\x01"sv),
+TestCase("%lld", "7846457973737734692", "%llu", "7846457973737734692", "\xc8\x88\xda\xda\x8d\xfd\x9b\xe4\xd9\x01"sv),
+TestCase("%lld", "749331093609658474", "%llu", "749331093609658474", "\xd4\xe1\x91\xdc\x9a\xaa\x94\xe6\x14"sv),
+TestCase("%lld", "4154208822466872538", "%llu", "4154208822466872538", "\xb4\x93\xf4\x9e\xc0\xba\xdb\xa6\x73"sv),
+TestCase("%lld", "8161063962699005564", "%llu", "8161063962699005564", "\xf8\xf9\xd3\xde\xea\x9b\xf6\xc1\xe2\x01"sv),
+TestCase("%lld", "3729503431677340594", "%llu", "3729503431677340594", "\xe4\xae\xd3\x9b\x90\xe7\xed\xc1\x67"sv),
+TestCase("%lld", "3621720449882747425", "%llu", "3621720449882747425", "\xc2\xc8\xbb\xb0\xef\xe3\xf7\xc2\x64"sv),
+TestCase("%lld", "3654452981777753952", "%llu", "3654452981777753952", "\xc0\x9d\xd7\xb3\xfe\xe7\x9c\xb7\x65"sv),
+TestCase("%lld", "8747087227033699739", "%llu", "8747087227033699739", "\xb6\x96\x9a\xab\xa5\xdf\xf2\xe3\xf2\x01"sv),
+TestCase("%lld", "7841542346336811454", "%llu", "7841542346336811454", "\xfc\xe6\xb5\xc9\xf7\xcd\xe0\xd2\xd9\x01"sv),
+TestCase("%lld", "8253451662283365177", "%llu", "8253451662283365177", "\xf2\xec\x8e\xd4\xdc\xa3\x93\x8a\xe5\x01"sv),
+TestCase("%lld", "3367538680468566654", "%llu", "3367538680468566654", "\xfc\xc9\xbb\x8b\xd0\xa5\xf3\xbb\x5d"sv),
+TestCase("%lld", "134795648044988988", "%llu", "134795648044988988", "\xf8\xe8\xeb\xef\xaf\xfb\xf1\xde\x03"sv),
+TestCase("%lld", "2210978119973242473", "%llu", "2210978119973242473", "\xd2\x99\xd4\xf2\xd8\xb8\xfc\xae\x3d"sv),
+TestCase("%lld", "421750780720578633", "%llu", "421750780720578633", "\x92\x81\xc0\xc9\xb7\x86\xae\xda\x0b"sv),
+TestCase("%lld", "5885301061939406661", "%llu", "5885301061939406661", "\x8a\x9d\x85\x92\xcb\x88\xe5\xac\xa3\x01"sv),
+TestCase("%lld", "8457438023241447265", "%llu", "8457438023241447265", "\xc2\xdd\x81\xd3\xd3\xc4\xed\xde\xea\x01"sv),
+TestCase("%lld", "8625342049665019705", "%llu", "8625342049665019705", "\xf2\xbc\xaf\x88\xa5\xb9\xaf\xb3\xef\x01"sv),
+TestCase("%lld", "4522738759120336903", "%llu", "4522738759120336903", "\x8e\xe0\x91\xd4\x8b\xbc\xff\xc3\x7d"sv),
+TestCase("%lld", "2671059116919281788", "%llu", "2671059116919281788", "\xf8\xe1\xd6\xe0\xad\x87\xc1\x91\x4a"sv),
+TestCase("%lld", "3110596655769706088", "%llu", "3110596655769706088", "\xd0\xf9\xde\xaa\xb0\xcb\x87\xab\x56"sv),
+TestCase("%lld", "7187907223546304710", "%llu", "7187907223546304710", "\x8c\xe3\x88\x9f\xac\xe2\xc9\xc0\xc7\x01"sv),
+TestCase("%lld", "6450021345187045785", "%llu", "6450021345187045785", "\xb2\x86\xfd\x98\xac\x8f\x8a\x83\xb3\x01"sv),
+TestCase("%lld", "1678455339349899965", "%llu", "1678455339349899965", "\xfa\xea\xf7\x94\xa0\x8f\x89\xcb\x2e"sv),
+TestCase("%lld", "8586083292869710126", "%llu", "8586083292869710126", "\xdc\xc4\xd2\xb6\xf1\xd0\xf2\xa7\xee\x01"sv),
+TestCase("%lld", "7328869729981674031", "%llu", "7328869729981674031", "\xde\xc8\xdd\xca\x8f\x8c\xb0\xb5\xcb\x01"sv),
+TestCase("%lld", "88833738086841272", "%llu", "88833738086841272", "\xf0\xee\xe3\xab\x8d\xf4\xcc\xbb\x02"sv),
+TestCase("%lld", "3875204489276489361", "%llu", "3875204489276489361", "\xa2\xfa\x97\xf5\xf0\xfc\xbe\xc7\x6b"sv),
+TestCase("%lld", "349662797203771308", "%llu", "349662797203771308", "\xd8\xbe\x80\xa8\xc4\x9d\xa0\xda\x09"sv),
+TestCase("%lld", "7970924659150689441", "%llu", "7970924659150689441", "\xc2\xd2\xa0\x81\xdd\xef\xb4\x9e\xdd\x01"sv),
+TestCase("%lld", "163280929539230956", "%llu", "163280929539230956", "\xd8\xb3\xc0\xbc\xfd\xc8\x8b\xc4\x04"sv),
+TestCase("%lld", "1719947238548425297", "%llu", "1719947238548425297", "\xa2\xa9\xbe\xa1\xd3\xb9\xbd\xde\x2f"sv),
+TestCase("%lld", "4371241115470848126", "%llu", "4371241115470848126", "\xfc\x81\xc7\xb3\xcf\xa8\xe2\xa9\x79"sv),
+TestCase("%lld", "4845845044868885160", "%llu", "4845845044868885160", "\xd0\xaa\xf9\xff\x9a\x99\xf3\xbf\x86\x01"sv),
+TestCase("%lld", "428374782176628108", "%llu", "428374782176628108", "\x98\xb6\xfc\xa5\x88\xa6\xf2\xf1\x0b"sv),
+TestCase("%lld", "7356256820074242781", "%llu", "7356256820074242781", "\xba\x8b\x8f\xd1\xc9\xa6\xd6\x96\xcc\x01"sv),
+TestCase("%lld", "7023621621475593673", "%llu", "7023621621475593673", "\x92\xb7\xf2\x8c\xdd\xa9\xf5\xf8\xc2\x01"sv),
+TestCase("%lld", "844047554037800681", "%llu", "844047554037800681", "\xd2\xfb\xae\xff\xe8\xb1\xd4\xb6\x17"sv),
+TestCase("%lld", "817568236458138451", "%llu", "817568236458138451", "\xa6\xed\xd6\x9b\xd7\xfe\xca\xd8\x16"sv),
+TestCase("%lld", "8750594193206555908", "%llu", "8750594193206555908", "\x88\xd4\xe1\xba\xcb\xc3\xad\xf0\xf2\x01"sv),
+TestCase("%lld", "8310288564467194046", "%llu", "8310288564467194046", "\xfc\xe2\xf6\xa0\xed\xda\x89\xd4\xe6\x01"sv),
+TestCase("%lld", "6530303901192542500", "%llu", "6530303901192542500", "\xc8\xe4\xee\x9e\x92\xb3\xa6\xa0\xb5\x01"sv),
+TestCase("%lld", "7528511123548308292", "%llu", "7528511123548308292", "\x88\x8d\xe4\xea\xd4\xbe\xd2\xfa\xd0\x01"sv),
+TestCase("%lld", "9162331973142792201", "%llu", "9162331973142792201", "\x92\xa0\x8a\xd6\xd5\x98\x92\xa7\xfe\x01"sv),
+TestCase("%lld", "2011986664423345991", "%llu", "2011986664423345991", "\x8e\xbd\xb6\x9c\xc3\xcd\x81\xec\x37"sv),
+TestCase("%lld", "9019302408819509", "%llu", "9019302408819509", "\xea\xcc\x98\xc3\xbf\xc0\x85\x20"sv),
+TestCase("%lld", "789517241633323881", "%llu", "789517241633323881", "\xd2\xed\xcd\xb9\xf1\xef\xf6\xf4\x15"sv),
+TestCase("%lld", "5000802077860900174", "%llu", "5000802077860900174", "\x9c\xb5\xf9\xae\xd2\xbf\xb5\xe6\x8a\x01"sv),
+TestCase("%lld", "634081739389621380", "%llu", "634081739389621380", "\x88\xd2\x8a\xae\xf0\xfe\xda\xcc\x11"sv),
+TestCase("%lld", "112762393776366951", "%llu", "112762393776366951", "\xce\x95\x81\x91\x97\xb3\xce\x90\x03"sv),
+TestCase("%lld", "6054079168136366365", "%llu", "6054079168136366365", "\xba\xa4\xe2\xad\xae\xbb\xb4\x84\xa8\x01"sv),
+TestCase("%lld", "3980060469571494571", "%llu", "3980060469571494571", "\xd6\x8a\x91\x96\x9d\xfa\x81\xbc\x6e"sv),
+TestCase("%lld", "5966260566188732596", "%llu", "5966260566188732596", "\xe8\xc2\xcd\xd1\xfa\x97\xb5\xcc\xa5\x01"sv),
+TestCase("%lld", "7994806728603097981", "%llu", "7994806728603097981", "\xfa\xed\xd2\xa4\x90\x97\xa1\xf3\xdd\x01"sv),
+TestCase("%lld", "4215789446045917951", "%llu", "4215789446045917951", "\xfe\x8b\xc3\xb6\xc7\x8a\xbf\x81\x75"sv),
+TestCase("%lld", "3106694682520283096", "%llu", "3106694682520283096", "\xb0\x9f\xbb\x93\xd2\x96\x99\x9d\x56"sv),
+TestCase("%lld", "7417099544643803895", "%llu", "7417099544643803895", "\xee\xeb\xa4\xe3\xa0\xaf\xea\xee\xcd\x01"sv),
+TestCase("%lld", "2442715921212337368", "%llu", "2442715921212337368", "\xb0\xb3\xeb\xb0\x86\xcc\xa2\xe6\x43"sv),
+TestCase("%lld", "3969107660140201186", "%llu", "3969107660140201186", "\xc4\x93\x82\xdf\xe7\x98\x8d\x95\x6e"sv),
+TestCase("%lld", "2820313669595847223", "%llu", "2820313669595847223", "\xee\xe8\x82\xf7\xdf\x95\xe2\xa3\x4e"sv),
+TestCase("%lld", "1153942061124594568", "%llu", "1153942061124594568", "\x90\xae\xb5\xcb\x9a\x8c\xd0\x83\x20"sv),
+TestCase("%lld", "7255377737678567782", "%llu", "7255377737678567782", "\xcc\xe5\xdd\xd9\x93\xe7\xa3\xb0\xc9\x01"sv),
+TestCase("%lld", "8010794524226708564", "%llu", "8010794524226708564", "\xa8\x91\xeb\x92\xa8\xcb\x87\xac\xde\x01"sv),
+TestCase("%lld", "2397984212273136806", "%llu", "2397984212273136806", "\xcc\x92\xc8\xd8\xf3\xfb\xac\xc7\x42"sv),
+TestCase("%lld", "3911421735689494721", "%llu", "3911421735689494721", "\x82\x83\x9d\xa8\x8a\xd6\x94\xc8\x6c"sv),
+TestCase("%lld", "8110862279364352221", "%llu", "8110862279364352221", "\xba\xe3\x84\xfa\xa2\x91\xc9\x8f\xe1\x01"sv),
+TestCase("%lld", "8302240689147774718", "%llu", "8302240689147774718", "\xfc\x9b\x94\xc7\xed\xfa\xbd\xb7\xe6\x01"sv),
+TestCase("%lld", "8137682451091990657", "%llu", "8137682451091990657", "\x82\xa2\xa5\x84\xde\xc4\xed\xee\xe1\x01"sv),
+TestCase("%lld", "7692576112763082769", "%llu", "7692576112763082769", "\xa2\xe0\xb0\x8e\xf6\xcd\xc2\xc1\xd5\x01"sv),
+TestCase("%lld", "4332208573039164264", "%llu", "4332208573039164264", "\xd0\xed\xb5\x8e\xd0\xaf\x8c\x9f\x78"sv),
+TestCase("%lld", "1408790260442838317", "%llu", "1408790260442838317", "\xda\xd4\xcd\xcb\xe3\xd1\x83\x8d\x27"sv),
+TestCase("%lld", "674974620879983391", "%llu", "674974620879983391", "\xbe\xcc\xa4\xdf\xed\xf5\xfe\xdd\x12"sv),
+TestCase("%lld", "2717882686191243732", "%llu", "2717882686191243732", "\xa8\xc7\xe8\xb9\xe6\xf9\xed\xb7\x4b"sv),
+TestCase("%lld", "2252874181081070317", "%llu", "2252874181081070317", "\xda\xeb\xe3\xef\xb4\xc8\xe8\xc3\x3e"sv),
+TestCase("%lld", "561600661159813802", "%llu", "561600661159813802", "\xd4\x8a\xf1\xa0\xed\xb4\x9a\xcb\x0f"sv),
+TestCase("%lld", "1743861870362420461", "%llu", "1743861870362420461", "\xda\x93\xe7\x84\xb7\xc8\xb8\xb3\x30"sv),
+TestCase("%lld", "469546530241540223", "%llu", "469546530241540223", "\xfe\xc1\x83\xc9\x9b\x85\x95\x84\x0d"sv),
+TestCase("%lld", "2803811754125283747", "%llu", "2803811754125283747", "\xc6\xc6\x9e\xdb\xec\xfb\x91\xe9\x4d"sv),
+TestCase("%lld", "1450094191759522064", "%llu", "1450094191759522064", "\xa0\xe4\xf7\xea\x80\xbf\xe2\x9f\x28"sv),
+TestCase("%lld", "973928254324237827", "%llu", "973928254324237827", "\x86\x98\xb3\xbb\xca\xa5\x8b\x84\x1b"sv),
+TestCase("%lld", "5007711289774136962", "%llu", "5007711289774136962", "\x84\xfa\x9a\xcb\xda\xb8\xfb\xfe\x8a\x01"sv),
+TestCase("%lld", "5617796192393039749", "%llu", "5617796192393039749", "\x8a\xbe\x88\xc2\xec\xf7\xb5\xf6\x9b\x01"sv),
+TestCase("%lld", "7609052204185497138", "%llu", "7609052204185497138", "\xe4\xb8\x89\xbe\xc9\xaa\xe4\x98\xd3\x01"sv),
+TestCase("%lld", "4109234434773033977", "%llu", "4109234434773033977", "\xf2\x8f\x99\xf7\xcb\xbc\xf7\x86\x72"sv),
+TestCase("%lld", "6196547806636196643", "%llu", "6196547806636196643", "\xc6\x8c\xb6\x84\xc8\xd9\xc7\xfe\xab\x01"sv),
+TestCase("%lld", "1242378025984660164", "%llu", "1242378025984660164", "\x88\xcb\x88\xb9\xee\x8e\xe8\xbd\x22"sv),
+TestCase("%lld", "3684142195824821322", "%llu", "3684142195824821322", "\x94\x91\xd4\xc2\xd8\xf3\xd9\xa0\x66"sv),
+TestCase("%lld", "7118933327258815856", "%llu", "7118933327258815856", "\xe0\xb5\x9c\xef\x96\x89\xc4\xcb\xc5\x01"sv),
+TestCase("%lld", "230557021351043665", "%llu", "230557021351043665", "\xa2\x89\xcf\xe1\xf5\x98\x8d\xb3\x06"sv),
+TestCase("%lld", "8445293705764922090", "%llu", "8445293705764922090", "\xd4\x8b\xaf\xbf\xab\xf8\xda\xb3\xea\x01"sv),
+TestCase("%lld", "1574191283697066832", "%llu", "1574191283697066832", "\xa0\x8d\xbd\x94\xba\xa8\xd3\xd8\x2b"sv),
+TestCase("%lld", "6141978416086614643", "%llu", "6141978416086614643", "\xe6\x89\xac\xcf\xfd\xb4\xd8\xbc\xaa\x01"sv),
+TestCase("%lld", "2281449725566748391", "%llu", "2281449725566748391", "\xce\x9b\x8c\x95\x82\x9c\xab\xa9\x3f"sv),
+TestCase("%lld", "784722535322631703", "%llu", "784722535322631703", "\xae\xf8\xb1\xd2\x9f\xbf\xf2\xe3\x15"sv),
+TestCase("%lld", "5758531472775216899", "%llu", "5758531472775216899", "\x86\xfc\xe3\xea\xa9\xf7\xb4\xea\x9f\x01"sv),
+TestCase("%lld", "3230692938369337673", "%llu", "3230692938369337673", "\x92\xa5\xf0\x9d\x89\x87\xdd\xd5\x59"sv),
+TestCase("%lld", "6060043374493141457", "%llu", "6060043374493141457", "\xa2\xa7\xfc\xc1\xee\xd5\xcc\x99\xa8\x01"sv),
+TestCase("%lld", "2194452418321024659", "%llu", "2194452418321024659", "\xa6\x9a\xac\xe8\xa0\xb6\xa1\xf4\x3c"sv),
+TestCase("%lld", "1179720724324426892", "%llu", "1179720724324426892", "\x98\x92\xb1\xb8\xf2\xef\x9a\xdf\x20"sv),
+TestCase("%lld", "6869292628752205971", "%llu", "6869292628752205971", "\xa6\xa2\xe0\xaf\x86\xd0\xd0\xd4\xbe\x01"sv),
+TestCase("%lld", "66666931390606463", "%llu", "66666931390606463", "\xfe\xb1\xeb\xbc\x91\xce\xec\xec\x01"sv),
+
+// Random 32-bit ints
+TestCase("%d", "548530896", "%u", "548530896", "\xa0\xab\x8f\x8b\x04"sv),
+TestCase("%d", "821325877", "%u", "821325877", "\xea\xc0\xa3\x8f\x06"sv),
+TestCase("%d", "511383431", "%u", "511383431", "\x8e\xde\xd8\xe7\x03"sv),
+TestCase("%d", "73179717", "%u", "73179717", "\x8a\x89\xe5\x45"sv),
+TestCase("%d", "644966342", "%u", "644966342", "\x8c\x9f\x8b\xe7\x04"sv),
+TestCase("%d", "1980405737", "%u", "1980405737", "\xd2\xdf\xd4\xe0\x0e"sv),
+TestCase("%d", "476646907", "%u", "476646907", "\xf6\xb7\xc8\xc6\x03"sv),
+TestCase("%d", "2032236920", "%u", "2032236920", "\xf0\xe5\x8b\x92\x0f"sv),
+TestCase("%d", "1293986078", "%u", "1293986078", "\xbc\xa4\x85\xd2\x09"sv),
+TestCase("%d", "301233235", "%u", "301233235", "\xa6\xd1\xa3\x9f\x02"sv),
+TestCase("%d", "558217992", "%u", "558217992", "\x90\xec\xad\x94\x04"sv),
+TestCase("%d", "2097556487", "%u", "2097556487", "\x8e\xb0\xb1\xd0\x0f"sv),
+TestCase("%d", "1914275808", "%u", "1914275808", "\xc0\x9f\xcc\xa1\x0e"sv),
+TestCase("%d", "817548252", "%u", "817548252", "\xb8\xaf\xd6\x8b\x06"sv),
+TestCase("%d", "409952181", "%u", "409952181", "\xea\xfe\xfa\x86\x03"sv),
+TestCase("%d", "1247456066", "%u", "1247456066", "\x84\xad\xd5\xa5\x09"sv),
+TestCase("%d", "1516178395", "%u", "1516178395", "\xb6\xaf\xf8\xa5\x0b"sv),
+TestCase("%d", "119460598", "%u", "119460598", "\xec\xcb\xf6\x71"sv),
+TestCase("%d", "1712502589", "%u", "1712502589", "\xfa\xdc\x95\xe1\x0c"sv),
+TestCase("%d", "1688223314", "%u", "1688223314", "\xa4\xf9\x81\xca\x0c"sv),
+TestCase("%d", "706807765", "%u", "706807765", "\xaa\x9f\x88\xa2\x05"sv),
+TestCase("%d", "1774808552", "%u", "1774808552", "\xd0\xb7\xcb\x9c\x0d"sv),
+TestCase("%d", "1789229650", "%u", "1789229650", "\xa4\xe9\xab\xaa\x0d"sv),
+TestCase("%d", "1849514292", "%u", "1849514292", "\xe8\xe4\xea\xe3\x0d"sv),
+TestCase("%d", "1655538874", "%u", "1655538874", "\xf4\x92\xec\xaa\x0c"sv),
+TestCase("%d", "34451703", "%u", "34451703", "\xee\xc3\xed\x20"sv),
+TestCase("%d", "1217027822", "%u", "1217027822", "\xdc\xfb\xd2\x88\x09"sv),
+TestCase("%d", "1722335750", "%u", "1722335750", "\x8c\x88\xc6\xea\x0c"sv),
+TestCase("%d", "1755007515", "%u", "1755007515", "\xb6\xa8\xda\x89\x0d"sv),
+TestCase("%d", "1016776167", "%u", "1016776167", "\xce\x97\xd6\xc9\x07"sv),
+TestCase("%d", "2053515555", "%u", "2053515555", "\xc6\xa4\xb1\xa6\x0f"sv),
+TestCase("%d", "652344632", "%u", "652344632", "\xf0\xf4\x8f\xee\x04"sv),
+TestCase("%d", "2075650855", "%u", "2075650855", "\xce\xac\xbf\xbb\x0f"sv),
+TestCase("%d", "1374899794", "%u", "1374899794", "\xa4\xb9\x9a\x9f\x0a"sv),
+TestCase("%d", "631591603", "%u", "631591603", "\xe6\xca\xaa\xda\x04"sv),
+TestCase("%d", "1017271797", "%u", "1017271797", "\xea\xd7\x92\xca\x07"sv),
+TestCase("%d", "659199737", "%u", "659199737", "\xf2\xdb\xd4\xf4\x04"sv),
+TestCase("%d", "1782422140", "%u", "1782422140", "\xf8\xe9\xec\xa3\x0d"sv),
+TestCase("%d", "1572071831", "%u", "1572071831", "\xae\xa6\x9f\xdb\x0b"sv),
+TestCase("%d", "2105608301", "%u", "2105608301", "\xda\xa1\x88\xd8\x0f"sv),
+TestCase("%d", "1534509283", "%u", "1534509283", "\xc6\x83\xb6\xb7\x0b"sv),
+TestCase("%d", "1360016822", "%u", "1360016822", "\xec\xd6\x81\x91\x0a"sv),
+TestCase("%d", "66588090", "%u", "66588090", "\xf4\xb6\xc0\x3f"sv),
+TestCase("%d", "2010287594", "%u", "2010287594", "\xd4\xb7\x94\xfd\x0e"sv),
+TestCase("%d", "1409055793", "%u", "1409055793", "\xe2\xf0\xe3\xbf\x0a"sv),
+TestCase("%d", "1312806256", "%u", "1312806256", "\xe0\xd5\xfe\xe3\x09"sv),
+TestCase("%d", "117699373", "%u", "117699373", "\xda\xcc\x9f\x70"sv),
+TestCase("%d", "1875416592", "%u", "1875416592", "\xa0\xd8\xc4\xfc\x0d"sv),
+TestCase("%d", "550403142", "%u", "550403142", "\x8c\xf1\xf3\x8c\x04"sv),
+TestCase("%d", "1556101213", "%u", "1556101213", "\xba\xe1\x81\xcc\x0b"sv),
+TestCase("%d", "1397393149", "%u", "1397393149", "\xfa\x9b\xd4\xb4\x0a"sv),
+TestCase("%d", "504462025", "%u", "504462025", "\x92\xeb\x8b\xe1\x03"sv),
+TestCase("%d", "1578855448", "%u", "1578855448", "\xb0\xb0\xdb\xe1\x0b"sv),
+TestCase("%d", "341299952", "%u", "341299952", "\xe0\xcb\xbe\xc5\x02"sv),
+TestCase("%d", "554765077", "%u", "554765077", "\xaa\xac\x88\x91\x04"sv),
+TestCase("%d", "1581023732", "%u", "1581023732", "\xe8\x87\xe4\xe3\x0b"sv),
+TestCase("%d", "1415869606", "%u", "1415869606", "\xcc\xd2\xa3\xc6\x0a"sv),
+TestCase("%d", "724653249", "%u", "724653249", "\x82\xd3\x8a\xb3\x05"sv),
+TestCase("%d", "88302425", "%u", "88302425", "\xb2\x8d\x9b\x54"sv),
+TestCase("%d", "1742669733", "%u", "1742669733", "\xca\x9e\xf8\xfd\x0c"sv),
+TestCase("%d", "878166891", "%u", "878166891", "\xd6\x8d\xbe\xc5\x06"sv),
+TestCase("%d", "1416428619", "%u", "1416428619", "\x96\xf1\xe7\xc6\x0a"sv),
+TestCase("%d", "1034414669", "%u", "1034414669", "\x9a\xa9\xbf\xda\x07"sv),
+TestCase("%d", "905412084", "%u", "905412084", "\xe8\xf7\xbb\xdf\x06"sv),
+TestCase("%d", "63837368", "%u", "63837368", "\xf0\xd2\xf0\x3c"sv),
+TestCase("%d", "645161396", "%u", "645161396", "\xe8\x86\xa3\xe7\x04"sv),
+TestCase("%d", "2107597739", "%u", "2107597739", "\xd6\x8e\xfb\xd9\x0f"sv),
+TestCase("%d", "76301137", "%u", "76301137", "\xa2\x8d\xe2\x48"sv),
+TestCase("%d", "459966688", "%u", "459966688", "\xc0\xa3\xd4\xb6\x03"sv),
+TestCase("%d", "1966018032", "%u", "1966018032", "\xe0\xb7\xf8\xd2\x0e"sv),
+TestCase("%d", "1781593154", "%u", "1781593154", "\x84\xd1\x87\xa3\x0d"sv),
+TestCase("%d", "558910911", "%u", "558910911", "\xfe\xb6\x82\x95\x04"sv),
+TestCase("%d", "1968681756", "%u", "1968681756", "\xb8\xcc\xbd\xd5\x0e"sv),
+TestCase("%d", "1898903331", "%u", "1898903331", "\xc6\xdc\xf7\x92\x0e"sv),
+TestCase("%d", "667424750", "%u", "667424750", "\xdc\xdf\xc0\xfc\x04"sv),
+TestCase("%d", "1831749058", "%u", "1831749058", "\x84\x97\xf2\xd2\x0d"sv),
+TestCase("%d", "1630416710", "%u", "1630416710", "\x8c\xbd\xf1\x92\x0c"sv),
+TestCase("%d", "1774702744", "%u", "1774702744", "\xb0\xc2\xbe\x9c\x0d"sv),
+TestCase("%d", "46739084", "%u", "46739084", "\x98\xba\xc9\x2c"sv),
+TestCase("%d", "709255105", "%u", "709255105", "\x82\xff\xb2\xa4\x05"sv),
+TestCase("%d", "1264949590", "%u", "1264949590", "\xac\xe5\xac\xb6\x09"sv),
+TestCase("%d", "2785021", "%u", "2785021", "\xfa\xfb\xd3\x02"sv),
+TestCase("%d", "1940076275", "%u", "1940076275", "\xe6\xdb\x99\xba\x0e"sv),
+TestCase("%d", "1436099990", "%u", "1436099990", "\xac\x96\xc9\xd9\x0a"sv),
+TestCase("%d", "622422778", "%u", "622422778", "\xf4\xab\xcb\xd1\x04"sv),
+TestCase("%d", "506192594", "%u", "506192594", "\xa4\x8b\xdf\xe2\x03"sv),
+TestCase("%d", "1329083686", "%u", "1329083686", "\xcc\xd4\xc1\xf3\x09"sv),
+TestCase("%d", "724686473", "%u", "724686473", "\x92\xda\x8e\xb3\x05"sv),
+TestCase("%d", "1693465014", "%u", "1693465014", "\xec\xe6\x81\xcf\x0c"sv),
+TestCase("%d", "986175143", "%u", "986175143", "\xce\xda\xbe\xac\x07"sv),
+TestCase("%d", "165118383", "%u", "165118383", "\xde\x86\xbc\x9d\x01"sv),
+TestCase("%d", "1364735728", "%u", "1364735728", "\xe0\xdb\xc1\x95\x0a"sv),
+TestCase("%d", "426265571", "%u", "426265571", "\xc6\xaf\xc2\x96\x03"sv),
+TestCase("%d", "1019467505", "%u", "1019467505", "\xe2\xdb\x9e\xcc\x07"sv),
+TestCase("%d", "353536599", "%u", "353536599", "\xae\xa9\x94\xd1\x02"sv),
+TestCase("%d", "1275925671", "%u", "1275925671", "\xce\xd2\xe8\xc0\x09"sv),
+TestCase("%d", "1250514267", "%u", "1250514267", "\xb6\xd5\xca\xa8\x09"sv),
+TestCase("%d", "1600951834", "%u", "1600951834", "\xb4\xd8\xe4\xf6\x0b"sv),
+TestCase("%d", "799391776", "%u", "799391776", "\xc0\x80\xae\xfa\x05"sv),
+TestCase("%d", "1812577120", "%u", "1812577120", "\xc0\xed\xcd\xc0\x0d"sv),
+
+// Random 16-bit ints
+TestCase("%d", "12031", "%u", "12031", "\xfe\xbb\x01"sv),
+TestCase("%d", "17500", "%u", "17500", "\xb8\x91\x02"sv),
+TestCase("%d", "14710", "%u", "14710", "\xec\xe5\x01"sv),
+TestCase("%d", "8546", "%u", "8546", "\xc4\x85\x01"sv),
+TestCase("%d", "25208", "%u", "25208", "\xf0\x89\x03"sv),
+TestCase("%d", "29977", "%u", "29977", "\xb2\xd4\x03"sv),
+TestCase("%d", "31629", "%u", "31629", "\x9a\xee\x03"sv),
+TestCase("%d", "28612", "%u", "28612", "\x88\xbf\x03"sv),
+TestCase("%d", "4804", "%u", "4804", "\x88\x4b"sv),
+TestCase("%d", "7142", "%u", "7142", "\xcc\x6f"sv),
+TestCase("%d", "18567", "%u", "18567", "\x8e\xa2\x02"sv),
+TestCase("%d", "9376", "%u", "9376", "\xc0\x92\x01"sv),
+TestCase("%d", "19912", "%u", "19912", "\x90\xb7\x02"sv),
+TestCase("%d", "18", "%u", "18", "\x24"sv),
+TestCase("%d", "12097", "%u", "12097", "\x82\xbd\x01"sv),
+TestCase("%d", "4004", "%u", "4004", "\xc8\x3e"sv),
+TestCase("%d", "13241", "%u", "13241", "\xf2\xce\x01"sv),
+TestCase("%d", "17823", "%u", "17823", "\xbe\x96\x02"sv),
+TestCase("%d", "11129", "%u", "11129", "\xf2\xad\x01"sv),
+TestCase("%d", "16567", "%u", "16567", "\xee\x82\x02"sv),
+TestCase("%d", "9138", "%u", "9138", "\xe4\x8e\x01"sv),
+TestCase("%d", "31404", "%u", "31404", "\xd8\xea\x03"sv),
+TestCase("%d", "1072", "%u", "1072", "\xe0\x10"sv),
+TestCase("%d", "12870", "%u", "12870", "\x8c\xc9\x01"sv),
+TestCase("%d", "25499", "%u", "25499", "\xb6\x8e\x03"sv),
+TestCase("%d", "1640", "%u", "1640", "\xd0\x19"sv),
+TestCase("%d", "26764", "%u", "26764", "\x98\xa2\x03"sv),
+TestCase("%d", "2078", "%u", "2078", "\xbc\x20"sv),
+TestCase("%d", "10264", "%u", "10264", "\xb0\xa0\x01"sv),
+TestCase("%d", "15533", "%u", "15533", "\xda\xf2\x01"sv),
+TestCase("%d", "7701", "%u", "7701", "\xaa\x78"sv),
+TestCase("%d", "24302", "%u", "24302", "\xdc\xfb\x02"sv),
+TestCase("%d", "10568", "%u", "10568", "\x90\xa5\x01"sv),
+TestCase("%d", "2206", "%u", "2206", "\xbc\x22"sv),
+TestCase("%d", "16240", "%u", "16240", "\xe0\xfd\x01"sv),
+TestCase("%d", "16600", "%u", "16600", "\xb0\x83\x02"sv),
+TestCase("%d", "30984", "%u", "30984", "\x90\xe4\x03"sv),
+TestCase("%d", "18489", "%u", "18489", "\xf2\xa0\x02"sv),
+TestCase("%d", "8613", "%u", "8613", "\xca\x86\x01"sv),
+TestCase("%d", "26441", "%u", "26441", "\x92\x9d\x03"sv),
+TestCase("%d", "22831", "%u", "22831", "\xde\xe4\x02"sv),
+TestCase("%d", "2307", "%u", "2307", "\x86\x24"sv),
+TestCase("%d", "24179", "%u", "24179", "\xe6\xf9\x02"sv),
+TestCase("%d", "6400", "%u", "6400", "\x80\x64"sv),
+TestCase("%d", "4264", "%u", "4264", "\xd0\x42"sv),
+TestCase("%d", "20770", "%u", "20770", "\xc4\xc4\x02"sv),
+TestCase("%d", "24276", "%u", "24276", "\xa8\xfb\x02"sv),
+TestCase("%d", "27013", "%u", "27013", "\x8a\xa6\x03"sv),
+TestCase("%d", "30434", "%u", "30434", "\xc4\xdb\x03"sv),
+TestCase("%d", "8338", "%u", "8338", "\xa4\x82\x01"sv),
+TestCase("%d", "20544", "%u", "20544", "\x80\xc1\x02"sv),
+TestCase("%d", "22908", "%u", "22908", "\xf8\xe5\x02"sv),
+TestCase("%d", "7111", "%u", "7111", "\x8e\x6f"sv),
+TestCase("%d", "30015", "%u", "30015", "\xfe\xd4\x03"sv),
+TestCase("%d", "16241", "%u", "16241", "\xe2\xfd\x01"sv),
+TestCase("%d", "6262", "%u", "6262", "\xec\x61"sv),
+TestCase("%d", "30724", "%u", "30724", "\x88\xe0\x03"sv),
+TestCase("%d", "7761", "%u", "7761", "\xa2\x79"sv),
+TestCase("%d", "12306", "%u", "12306", "\xa4\xc0\x01"sv),
+TestCase("%d", "5472", "%u", "5472", "\xc0\x55"sv),
+TestCase("%d", "20833", "%u", "20833", "\xc2\xc5\x02"sv),
+TestCase("%d", "19726", "%u", "19726", "\x9c\xb4\x02"sv),
+TestCase("%d", "20971", "%u", "20971", "\xd6\xc7\x02"sv),
+TestCase("%d", "12697", "%u", "12697", "\xb2\xc6\x01"sv),
+TestCase("%d", "29872", "%u", "29872", "\xe0\xd2\x03"sv),
+TestCase("%d", "20930", "%u", "20930", "\x84\xc7\x02"sv),
+TestCase("%d", "11185", "%u", "11185", "\xe2\xae\x01"sv),
+TestCase("%d", "4894", "%u", "4894", "\xbc\x4c"sv),
+TestCase("%d", "21973", "%u", "21973", "\xaa\xd7\x02"sv),
+TestCase("%d", "32040", "%u", "32040", "\xd0\xf4\x03"sv),
+TestCase("%d", "26757", "%u", "26757", "\x8a\xa2\x03"sv),
+TestCase("%d", "18963", "%u", "18963", "\xa6\xa8\x02"sv),
+TestCase("%d", "338", "%u", "338", "\xa4\x05"sv),
+TestCase("%d", "18607", "%u", "18607", "\xde\xa2\x02"sv),
+TestCase("%d", "7633", "%u", "7633", "\xa2\x77"sv),
+TestCase("%d", "14357", "%u", "14357", "\xaa\xe0\x01"sv),
+TestCase("%d", "14672", "%u", "14672", "\xa0\xe5\x01"sv),
+TestCase("%d", "13025", "%u", "13025", "\xc2\xcb\x01"sv),
+TestCase("%d", "4413", "%u", "4413", "\xfa\x44"sv),
+TestCase("%d", "2946", "%u", "2946", "\x84\x2e"sv),
+TestCase("%d", "12641", "%u", "12641", "\xc2\xc5\x01"sv),
+TestCase("%d", "28130", "%u", "28130", "\xc4\xb7\x03"sv),
+TestCase("%d", "6892", "%u", "6892", "\xd8\x6b"sv),
+TestCase("%d", "27495", "%u", "27495", "\xce\xad\x03"sv),
+TestCase("%d", "11488", "%u", "11488", "\xc0\xb3\x01"sv),
+TestCase("%d", "23283", "%u", "23283", "\xe6\xeb\x02"sv),
+TestCase("%d", "29949", "%u", "29949", "\xfa\xd3\x03"sv),
+TestCase("%d", "5161", "%u", "5161", "\xd2\x50"sv),
+TestCase("%d", "10664", "%u", "10664", "\xd0\xa6\x01"sv),
+TestCase("%d", "22534", "%u", "22534", "\x8c\xe0\x02"sv),
+TestCase("%d", "6502", "%u", "6502", "\xcc\x65"sv),
+TestCase("%d", "28037", "%u", "28037", "\x8a\xb6\x03"sv),
+TestCase("%d", "305", "%u", "305", "\xe2\x04"sv),
+TestCase("%d", "32042", "%u", "32042", "\xd4\xf4\x03"sv),
+TestCase("%d", "8481", "%u", "8481", "\xc2\x84\x01"sv),
+TestCase("%d", "22068", "%u", "22068", "\xe8\xd8\x02"sv),
+TestCase("%d", "13788", "%u", "13788", "\xb8\xd7\x01"sv),
+TestCase("%d", "29904", "%u", "29904", "\xa0\xd3\x03"sv),
+TestCase("%d", "12689", "%u", "12689", "\xa2\xc6\x01"sv),
+TestCase("%d", "1205", "%u", "1205", "\xea\x12"sv),
+
+// All 8-bit numbers
+TestCase("%d", "-128", "%u", "4294967168", "\xff\x01"sv),
+TestCase("%d", "-127", "%u", "4294967169", "\xfd\x01"sv),
+TestCase("%d", "-126", "%u", "4294967170", "\xfb\x01"sv),
+TestCase("%d", "-125", "%u", "4294967171", "\xf9\x01"sv),
+TestCase("%d", "-124", "%u", "4294967172", "\xf7\x01"sv),
+TestCase("%d", "-123", "%u", "4294967173", "\xf5\x01"sv),
+TestCase("%d", "-122", "%u", "4294967174", "\xf3\x01"sv),
+TestCase("%d", "-121", "%u", "4294967175", "\xf1\x01"sv),
+TestCase("%d", "-120", "%u", "4294967176", "\xef\x01"sv),
+TestCase("%d", "-119", "%u", "4294967177", "\xed\x01"sv),
+TestCase("%d", "-118", "%u", "4294967178", "\xeb\x01"sv),
+TestCase("%d", "-117", "%u", "4294967179", "\xe9\x01"sv),
+TestCase("%d", "-116", "%u", "4294967180", "\xe7\x01"sv),
+TestCase("%d", "-115", "%u", "4294967181", "\xe5\x01"sv),
+TestCase("%d", "-114", "%u", "4294967182", "\xe3\x01"sv),
+TestCase("%d", "-113", "%u", "4294967183", "\xe1\x01"sv),
+TestCase("%d", "-112", "%u", "4294967184", "\xdf\x01"sv),
+TestCase("%d", "-111", "%u", "4294967185", "\xdd\x01"sv),
+TestCase("%d", "-110", "%u", "4294967186", "\xdb\x01"sv),
+TestCase("%d", "-109", "%u", "4294967187", "\xd9\x01"sv),
+TestCase("%d", "-108", "%u", "4294967188", "\xd7\x01"sv),
+TestCase("%d", "-107", "%u", "4294967189", "\xd5\x01"sv),
+TestCase("%d", "-106", "%u", "4294967190", "\xd3\x01"sv),
+TestCase("%d", "-105", "%u", "4294967191", "\xd1\x01"sv),
+TestCase("%d", "-104", "%u", "4294967192", "\xcf\x01"sv),
+TestCase("%d", "-103", "%u", "4294967193", "\xcd\x01"sv),
+TestCase("%d", "-102", "%u", "4294967194", "\xcb\x01"sv),
+TestCase("%d", "-101", "%u", "4294967195", "\xc9\x01"sv),
+TestCase("%d", "-100", "%u", "4294967196", "\xc7\x01"sv),
+TestCase("%d", "-99", "%u", "4294967197", "\xc5\x01"sv),
+TestCase("%d", "-98", "%u", "4294967198", "\xc3\x01"sv),
+TestCase("%d", "-97", "%u", "4294967199", "\xc1\x01"sv),
+TestCase("%d", "-96", "%u", "4294967200", "\xbf\x01"sv),
+TestCase("%d", "-95", "%u", "4294967201", "\xbd\x01"sv),
+TestCase("%d", "-94", "%u", "4294967202", "\xbb\x01"sv),
+TestCase("%d", "-93", "%u", "4294967203", "\xb9\x01"sv),
+TestCase("%d", "-92", "%u", "4294967204", "\xb7\x01"sv),
+TestCase("%d", "-91", "%u", "4294967205", "\xb5\x01"sv),
+TestCase("%d", "-90", "%u", "4294967206", "\xb3\x01"sv),
+TestCase("%d", "-89", "%u", "4294967207", "\xb1\x01"sv),
+TestCase("%d", "-88", "%u", "4294967208", "\xaf\x01"sv),
+TestCase("%d", "-87", "%u", "4294967209", "\xad\x01"sv),
+TestCase("%d", "-86", "%u", "4294967210", "\xab\x01"sv),
+TestCase("%d", "-85", "%u", "4294967211", "\xa9\x01"sv),
+TestCase("%d", "-84", "%u", "4294967212", "\xa7\x01"sv),
+TestCase("%d", "-83", "%u", "4294967213", "\xa5\x01"sv),
+TestCase("%d", "-82", "%u", "4294967214", "\xa3\x01"sv),
+TestCase("%d", "-81", "%u", "4294967215", "\xa1\x01"sv),
+TestCase("%d", "-80", "%u", "4294967216", "\x9f\x01"sv),
+TestCase("%d", "-79", "%u", "4294967217", "\x9d\x01"sv),
+TestCase("%d", "-78", "%u", "4294967218", "\x9b\x01"sv),
+TestCase("%d", "-77", "%u", "4294967219", "\x99\x01"sv),
+TestCase("%d", "-76", "%u", "4294967220", "\x97\x01"sv),
+TestCase("%d", "-75", "%u", "4294967221", "\x95\x01"sv),
+TestCase("%d", "-74", "%u", "4294967222", "\x93\x01"sv),
+TestCase("%d", "-73", "%u", "4294967223", "\x91\x01"sv),
+TestCase("%d", "-72", "%u", "4294967224", "\x8f\x01"sv),
+TestCase("%d", "-71", "%u", "4294967225", "\x8d\x01"sv),
+TestCase("%d", "-70", "%u", "4294967226", "\x8b\x01"sv),
+TestCase("%d", "-69", "%u", "4294967227", "\x89\x01"sv),
+TestCase("%d", "-68", "%u", "4294967228", "\x87\x01"sv),
+TestCase("%d", "-67", "%u", "4294967229", "\x85\x01"sv),
+TestCase("%d", "-66", "%u", "4294967230", "\x83\x01"sv),
+TestCase("%d", "-65", "%u", "4294967231", "\x81\x01"sv),
+TestCase("%d", "-64", "%u", "4294967232", "\x7f"sv),
+TestCase("%d", "-63", "%u", "4294967233", "\x7d"sv),
+TestCase("%d", "-62", "%u", "4294967234", "\x7b"sv),
+TestCase("%d", "-61", "%u", "4294967235", "\x79"sv),
+TestCase("%d", "-60", "%u", "4294967236", "\x77"sv),
+TestCase("%d", "-59", "%u", "4294967237", "\x75"sv),
+TestCase("%d", "-58", "%u", "4294967238", "\x73"sv),
+TestCase("%d", "-57", "%u", "4294967239", "\x71"sv),
+TestCase("%d", "-56", "%u", "4294967240", "\x6f"sv),
+TestCase("%d", "-55", "%u", "4294967241", "\x6d"sv),
+TestCase("%d", "-54", "%u", "4294967242", "\x6b"sv),
+TestCase("%d", "-53", "%u", "4294967243", "\x69"sv),
+TestCase("%d", "-52", "%u", "4294967244", "\x67"sv),
+TestCase("%d", "-51", "%u", "4294967245", "\x65"sv),
+TestCase("%d", "-50", "%u", "4294967246", "\x63"sv),
+TestCase("%d", "-49", "%u", "4294967247", "\x61"sv),
+TestCase("%d", "-48", "%u", "4294967248", "\x5f"sv),
+TestCase("%d", "-47", "%u", "4294967249", "\x5d"sv),
+TestCase("%d", "-46", "%u", "4294967250", "\x5b"sv),
+TestCase("%d", "-45", "%u", "4294967251", "\x59"sv),
+TestCase("%d", "-44", "%u", "4294967252", "\x57"sv),
+TestCase("%d", "-43", "%u", "4294967253", "\x55"sv),
+TestCase("%d", "-42", "%u", "4294967254", "\x53"sv),
+TestCase("%d", "-41", "%u", "4294967255", "\x51"sv),
+TestCase("%d", "-40", "%u", "4294967256", "\x4f"sv),
+TestCase("%d", "-39", "%u", "4294967257", "\x4d"sv),
+TestCase("%d", "-38", "%u", "4294967258", "\x4b"sv),
+TestCase("%d", "-37", "%u", "4294967259", "\x49"sv),
+TestCase("%d", "-36", "%u", "4294967260", "\x47"sv),
+TestCase("%d", "-35", "%u", "4294967261", "\x45"sv),
+TestCase("%d", "-34", "%u", "4294967262", "\x43"sv),
+TestCase("%d", "-33", "%u", "4294967263", "\x41"sv),
+TestCase("%d", "-32", "%u", "4294967264", "\x3f"sv),
+TestCase("%d", "-31", "%u", "4294967265", "\x3d"sv),
+TestCase("%d", "-30", "%u", "4294967266", "\x3b"sv),
+TestCase("%d", "-29", "%u", "4294967267", "\x39"sv),
+TestCase("%d", "-28", "%u", "4294967268", "\x37"sv),
+TestCase("%d", "-27", "%u", "4294967269", "\x35"sv),
+TestCase("%d", "-26", "%u", "4294967270", "\x33"sv),
+TestCase("%d", "-25", "%u", "4294967271", "\x31"sv),
+TestCase("%d", "-24", "%u", "4294967272", "\x2f"sv),
+TestCase("%d", "-23", "%u", "4294967273", "\x2d"sv),
+TestCase("%d", "-22", "%u", "4294967274", "\x2b"sv),
+TestCase("%d", "-21", "%u", "4294967275", "\x29"sv),
+TestCase("%d", "-20", "%u", "4294967276", "\x27"sv),
+TestCase("%d", "-19", "%u", "4294967277", "\x25"sv),
+TestCase("%d", "-18", "%u", "4294967278", "\x23"sv),
+TestCase("%d", "-17", "%u", "4294967279", "\x21"sv),
+TestCase("%d", "-16", "%u", "4294967280", "\x1f"sv),
+TestCase("%d", "-15", "%u", "4294967281", "\x1d"sv),
+TestCase("%d", "-14", "%u", "4294967282", "\x1b"sv),
+TestCase("%d", "-13", "%u", "4294967283", "\x19"sv),
+TestCase("%d", "-12", "%u", "4294967284", "\x17"sv),
+TestCase("%d", "-11", "%u", "4294967285", "\x15"sv),
+TestCase("%d", "-10", "%u", "4294967286", "\x13"sv),
+TestCase("%d", "-9", "%u", "4294967287", "\x11"sv),
+TestCase("%d", "-8", "%u", "4294967288", "\x0f"sv),
+TestCase("%d", "-7", "%u", "4294967289", "\x0d"sv),
+TestCase("%d", "-6", "%u", "4294967290", "\x0b"sv),
+TestCase("%d", "-5", "%u", "4294967291", "\x09"sv),
+TestCase("%d", "-4", "%u", "4294967292", "\x07"sv),
+TestCase("%d", "-3", "%u", "4294967293", "\x05"sv),
+TestCase("%d", "-2", "%u", "4294967294", "\x03"sv),
+TestCase("%d", "-1", "%u", "4294967295", "\x01"sv),
+TestCase("%d", "0", "%u", "0", "\x00"sv),
+TestCase("%d", "1", "%u", "1", "\x02"sv),
+TestCase("%d", "2", "%u", "2", "\x04"sv),
+TestCase("%d", "3", "%u", "3", "\x06"sv),
+TestCase("%d", "4", "%u", "4", "\x08"sv),
+TestCase("%d", "5", "%u", "5", "\x0a"sv),
+TestCase("%d", "6", "%u", "6", "\x0c"sv),
+TestCase("%d", "7", "%u", "7", "\x0e"sv),
+TestCase("%d", "8", "%u", "8", "\x10"sv),
+TestCase("%d", "9", "%u", "9", "\x12"sv),
+TestCase("%d", "10", "%u", "10", "\x14"sv),
+TestCase("%d", "11", "%u", "11", "\x16"sv),
+TestCase("%d", "12", "%u", "12", "\x18"sv),
+TestCase("%d", "13", "%u", "13", "\x1a"sv),
+TestCase("%d", "14", "%u", "14", "\x1c"sv),
+TestCase("%d", "15", "%u", "15", "\x1e"sv),
+TestCase("%d", "16", "%u", "16", "\x20"sv),
+TestCase("%d", "17", "%u", "17", "\x22"sv),
+TestCase("%d", "18", "%u", "18", "\x24"sv),
+TestCase("%d", "19", "%u", "19", "\x26"sv),
+TestCase("%d", "20", "%u", "20", "\x28"sv),
+TestCase("%d", "21", "%u", "21", "\x2a"sv),
+TestCase("%d", "22", "%u", "22", "\x2c"sv),
+TestCase("%d", "23", "%u", "23", "\x2e"sv),
+TestCase("%d", "24", "%u", "24", "\x30"sv),
+TestCase("%d", "25", "%u", "25", "\x32"sv),
+TestCase("%d", "26", "%u", "26", "\x34"sv),
+TestCase("%d", "27", "%u", "27", "\x36"sv),
+TestCase("%d", "28", "%u", "28", "\x38"sv),
+TestCase("%d", "29", "%u", "29", "\x3a"sv),
+TestCase("%d", "30", "%u", "30", "\x3c"sv),
+TestCase("%d", "31", "%u", "31", "\x3e"sv),
+TestCase("%d", "32", "%u", "32", "\x40"sv),
+TestCase("%d", "33", "%u", "33", "\x42"sv),
+TestCase("%d", "34", "%u", "34", "\x44"sv),
+TestCase("%d", "35", "%u", "35", "\x46"sv),
+TestCase("%d", "36", "%u", "36", "\x48"sv),
+TestCase("%d", "37", "%u", "37", "\x4a"sv),
+TestCase("%d", "38", "%u", "38", "\x4c"sv),
+TestCase("%d", "39", "%u", "39", "\x4e"sv),
+TestCase("%d", "40", "%u", "40", "\x50"sv),
+TestCase("%d", "41", "%u", "41", "\x52"sv),
+TestCase("%d", "42", "%u", "42", "\x54"sv),
+TestCase("%d", "43", "%u", "43", "\x56"sv),
+TestCase("%d", "44", "%u", "44", "\x58"sv),
+TestCase("%d", "45", "%u", "45", "\x5a"sv),
+TestCase("%d", "46", "%u", "46", "\x5c"sv),
+TestCase("%d", "47", "%u", "47", "\x5e"sv),
+TestCase("%d", "48", "%u", "48", "\x60"sv),
+TestCase("%d", "49", "%u", "49", "\x62"sv),
+TestCase("%d", "50", "%u", "50", "\x64"sv),
+TestCase("%d", "51", "%u", "51", "\x66"sv),
+TestCase("%d", "52", "%u", "52", "\x68"sv),
+TestCase("%d", "53", "%u", "53", "\x6a"sv),
+TestCase("%d", "54", "%u", "54", "\x6c"sv),
+TestCase("%d", "55", "%u", "55", "\x6e"sv),
+TestCase("%d", "56", "%u", "56", "\x70"sv),
+TestCase("%d", "57", "%u", "57", "\x72"sv),
+TestCase("%d", "58", "%u", "58", "\x74"sv),
+TestCase("%d", "59", "%u", "59", "\x76"sv),
+TestCase("%d", "60", "%u", "60", "\x78"sv),
+TestCase("%d", "61", "%u", "61", "\x7a"sv),
+TestCase("%d", "62", "%u", "62", "\x7c"sv),
+TestCase("%d", "63", "%u", "63", "\x7e"sv),
+TestCase("%d", "64", "%u", "64", "\x80\x01"sv),
+TestCase("%d", "65", "%u", "65", "\x82\x01"sv),
+TestCase("%d", "66", "%u", "66", "\x84\x01"sv),
+TestCase("%d", "67", "%u", "67", "\x86\x01"sv),
+TestCase("%d", "68", "%u", "68", "\x88\x01"sv),
+TestCase("%d", "69", "%u", "69", "\x8a\x01"sv),
+TestCase("%d", "70", "%u", "70", "\x8c\x01"sv),
+TestCase("%d", "71", "%u", "71", "\x8e\x01"sv),
+TestCase("%d", "72", "%u", "72", "\x90\x01"sv),
+TestCase("%d", "73", "%u", "73", "\x92\x01"sv),
+TestCase("%d", "74", "%u", "74", "\x94\x01"sv),
+TestCase("%d", "75", "%u", "75", "\x96\x01"sv),
+TestCase("%d", "76", "%u", "76", "\x98\x01"sv),
+TestCase("%d", "77", "%u", "77", "\x9a\x01"sv),
+TestCase("%d", "78", "%u", "78", "\x9c\x01"sv),
+TestCase("%d", "79", "%u", "79", "\x9e\x01"sv),
+TestCase("%d", "80", "%u", "80", "\xa0\x01"sv),
+TestCase("%d", "81", "%u", "81", "\xa2\x01"sv),
+TestCase("%d", "82", "%u", "82", "\xa4\x01"sv),
+TestCase("%d", "83", "%u", "83", "\xa6\x01"sv),
+TestCase("%d", "84", "%u", "84", "\xa8\x01"sv),
+TestCase("%d", "85", "%u", "85", "\xaa\x01"sv),
+TestCase("%d", "86", "%u", "86", "\xac\x01"sv),
+TestCase("%d", "87", "%u", "87", "\xae\x01"sv),
+TestCase("%d", "88", "%u", "88", "\xb0\x01"sv),
+TestCase("%d", "89", "%u", "89", "\xb2\x01"sv),
+TestCase("%d", "90", "%u", "90", "\xb4\x01"sv),
+TestCase("%d", "91", "%u", "91", "\xb6\x01"sv),
+TestCase("%d", "92", "%u", "92", "\xb8\x01"sv),
+TestCase("%d", "93", "%u", "93", "\xba\x01"sv),
+TestCase("%d", "94", "%u", "94", "\xbc\x01"sv),
+TestCase("%d", "95", "%u", "95", "\xbe\x01"sv),
+TestCase("%d", "96", "%u", "96", "\xc0\x01"sv),
+TestCase("%d", "97", "%u", "97", "\xc2\x01"sv),
+TestCase("%d", "98", "%u", "98", "\xc4\x01"sv),
+TestCase("%d", "99", "%u", "99", "\xc6\x01"sv),
+TestCase("%d", "100", "%u", "100", "\xc8\x01"sv),
+TestCase("%d", "101", "%u", "101", "\xca\x01"sv),
+TestCase("%d", "102", "%u", "102", "\xcc\x01"sv),
+TestCase("%d", "103", "%u", "103", "\xce\x01"sv),
+TestCase("%d", "104", "%u", "104", "\xd0\x01"sv),
+TestCase("%d", "105", "%u", "105", "\xd2\x01"sv),
+TestCase("%d", "106", "%u", "106", "\xd4\x01"sv),
+TestCase("%d", "107", "%u", "107", "\xd6\x01"sv),
+TestCase("%d", "108", "%u", "108", "\xd8\x01"sv),
+TestCase("%d", "109", "%u", "109", "\xda\x01"sv),
+TestCase("%d", "110", "%u", "110", "\xdc\x01"sv),
+TestCase("%d", "111", "%u", "111", "\xde\x01"sv),
+TestCase("%d", "112", "%u", "112", "\xe0\x01"sv),
+TestCase("%d", "113", "%u", "113", "\xe2\x01"sv),
+TestCase("%d", "114", "%u", "114", "\xe4\x01"sv),
+TestCase("%d", "115", "%u", "115", "\xe6\x01"sv),
+TestCase("%d", "116", "%u", "116", "\xe8\x01"sv),
+TestCase("%d", "117", "%u", "117", "\xea\x01"sv),
+TestCase("%d", "118", "%u", "118", "\xec\x01"sv),
+TestCase("%d", "119", "%u", "119", "\xee\x01"sv),
+TestCase("%d", "120", "%u", "120", "\xf0\x01"sv),
+TestCase("%d", "121", "%u", "121", "\xf2\x01"sv),
+TestCase("%d", "122", "%u", "122", "\xf4\x01"sv),
+TestCase("%d", "123", "%u", "123", "\xf6\x01"sv),
+TestCase("%d", "124", "%u", "124", "\xf8\x01"sv),
+TestCase("%d", "125", "%u", "125", "\xfa\x01"sv),
+TestCase("%d", "126", "%u", "126", "\xfc\x01"sv),
+TestCase("%d", "127", "%u", "127", "\xfe\x01"sv),
+
+};
+
+} // namespace pw::test::varint_decoding
diff --git a/pw_tokenizer/token_database.cc b/pw_tokenizer/token_database.cc
new file mode 100644
index 0000000..42c145b
--- /dev/null
+++ b/pw_tokenizer/token_database.cc
@@ -0,0 +1,41 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_tokenizer/token_database.h"
+
+namespace pw::tokenizer {
+
+TokenDatabase::Entry TokenDatabase::Entries::operator[](size_t index) const {
+ Iterator it = begin();
+ for (size_t i = 0; i < index; ++i) {
+ ++it;
+ }
+ return it.entry();
+}
+
+TokenDatabase::Entries TokenDatabase::Find(const uint32_t token) const {
+ Iterator first = begin();
+ while (first != end() && token > first->token) {
+ ++first;
+ }
+
+ Iterator last = first;
+ while (last != end() && token == last->token) {
+ ++last;
+ }
+
+ return Entries(first, last);
+}
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/token_database_test.cc b/pw_tokenizer/token_database_test.cc
new file mode 100644
index 0000000..0dccfb6
--- /dev/null
+++ b/pw_tokenizer/token_database_test.cc
@@ -0,0 +1,263 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_tokenizer/token_database.h"
+
+#include <cstring>
+#include <string>
+#include <string_view>
+
+#include "gtest/gtest.h"
+
+namespace pw::tokenizer {
+namespace {
+
+using namespace std::literals::string_view_literals;
+
+// Use alignas to ensure that the data is properly aligned for database entries.
+// This avoids unaligned memory reads.
+alignas(TokenDatabase::RawEntry) constexpr char kBasicData[] =
+ "TOKENS\0\0\x03\x00\x00\x00\0\0\0\0"
+ "\x01\0\0\0\0\0\0\0"
+ "\x02\0\0\0\0\0\0\0"
+ "\xFF\0\0\0\0\0\0\0"
+ "hi!\0"
+ "goodbye\0"
+ ":)";
+
+alignas(TokenDatabase::RawEntry) constexpr char kEmptyData[] =
+ "TOKENS\0\0\x00\x00\x00\x00\0\0\0"; // Last byte is null terminator.
+
+alignas(TokenDatabase::RawEntry) constexpr char kBadMagic[] =
+ "TOKENs\0\0\x03\x00\x00\x00\0\0\0\0"
+ "\x01\0\0\0\0\0\0\0"
+ "hi!\0";
+
+alignas(TokenDatabase::RawEntry) constexpr char kBadVersion[] =
+ "TOKENS\0\1\x00\0\0\0\0\0\0\0";
+
+alignas(TokenDatabase::RawEntry) constexpr char kBadEntryCount[] =
+ "TOKENS\0\0\xff\x00\x00\x00\0\0\0\0";
+
+// Use signed data and a size with the top bit set to test that the entry count
+// is read correctly, without per-byte sign extension.
+alignas(TokenDatabase::RawEntry) constexpr signed char kSignedWithTopBit[] =
+ "TOKENS\0\0\x80\x00\x00\x00\0\0\0\0"
+ // Entries
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate" // 32
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate" // 64
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate" // 96
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate"
+ "TOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdateTOKNdate" // 128
+ // Strings (empty)
+ "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" // 32
+ "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" // 64
+ "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0" // 96
+ "\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"; // 128
+
+constexpr TokenDatabase kBasicDatabase = TokenDatabase::Create<kBasicData>();
+static_assert(kBasicDatabase.size() == 3u);
+
+TEST(TokenDatabase, EntryCount) {
+ static_assert(TokenDatabase::Create<kBasicData>().size() == 3u);
+ static_assert(TokenDatabase::Create(kEmptyData).size() == 0u);
+ EXPECT_EQ(TokenDatabase::Create<kSignedWithTopBit>().size(), 0x80u);
+}
+
+TEST(TokenDatabase, ValidCheck) {
+ char basic_data[sizeof(kBasicData)];
+ std::memcpy(basic_data, kBasicData, sizeof(basic_data));
+ EXPECT_TRUE(TokenDatabase::IsValid(basic_data));
+
+ static_assert(TokenDatabase::IsValid(kBasicData));
+ static_assert(TokenDatabase::IsValid(kEmptyData));
+ static_assert(TokenDatabase::IsValid(kSignedWithTopBit));
+
+ static_assert(!TokenDatabase::IsValid(kBadMagic));
+ static_assert(!TokenDatabase::IsValid(kBadVersion));
+ static_assert(!TokenDatabase::IsValid(kBadEntryCount));
+
+ static_assert(!TokenDatabase::IsValid("TOKENS\0\0\0\0")); // too short
+ static_assert(!TokenDatabase::IsValid("TOKENS\0\1\0\0\0\0\0\0\0\0"));
+ static_assert(!TokenDatabase::IsValid("TOKENSv0\0\0\0\0\0\0\0\0"));
+ static_assert(!TokenDatabase::IsValid("tokens\0\0\0\0\0\0\0\0\0\0"));
+
+ // No string table; this is one byte too short.
+ static_assert(
+ !TokenDatabase::IsValid("TOKENS\0\0\x01\x00\x00\x00\0\0\0\0WXYZdate"sv));
+
+ // Add one byte for the string table.
+ static_assert(
+ TokenDatabase::IsValid("TOKENS\0\0\x01\x00\x00\x00\0\0\0\0WXYZdate\0"sv));
+
+ static_assert(
+ !TokenDatabase::IsValid("TOKENS\0\0\x02\x00\x00\x00\0\0\0\0"
+ "WXYZdate"
+ "WXYZdate"
+ "\0"sv));
+ static_assert(
+ TokenDatabase::IsValid("TOKENS\0\0\x02\x00\x00\x00\0\0\0\0"
+ "WXYZdate"
+ "WXYZdate"
+ "hi\0\0"sv));
+ static_assert(
+ TokenDatabase::IsValid("TOKENS\0\0\x02\x00\x00\x00\0\0\0\0"
+ "WXYZdate"
+ "WXYZdate"
+ "hi\0hello\0"sv));
+}
+
+TEST(TokenDatabase, Iterator) {
+ auto it = kBasicDatabase.begin();
+ EXPECT_EQ(it->token, 1u);
+ EXPECT_STREQ(it.entry().string, "hi!");
+
+ ++it;
+ EXPECT_EQ(it->token, 2u);
+ EXPECT_STREQ(it.entry().string, "goodbye");
+ EXPECT_EQ(it - kBasicDatabase.begin(), 1);
+
+ ++it;
+ EXPECT_EQ(it->token, 0xFFu);
+ EXPECT_STREQ(it.entry().string, ":)");
+ EXPECT_EQ(it - kBasicDatabase.begin(), 2);
+
+ ++it;
+ EXPECT_EQ(it, kBasicDatabase.end());
+ EXPECT_EQ(static_cast<size_t>(it - kBasicDatabase.begin()),
+ kBasicDatabase.size());
+}
+
+TEST(TokenDatabase, SingleEntryLookup_FirstEntry) {
+ auto match = kBasicDatabase.Find(1);
+ ASSERT_EQ(match.size(), 1u);
+ EXPECT_FALSE(match.empty());
+ EXPECT_STREQ(match[0].string, "hi!");
+
+ for (const auto& entry : match) {
+ EXPECT_EQ(entry.token, 1u);
+ EXPECT_STREQ(entry.string, "hi!");
+ }
+}
+
+TEST(TokenDatabase, SingleEntryLookup_MiddleEntry) {
+ auto match = kBasicDatabase.Find(2);
+ ASSERT_EQ(match.size(), 1u);
+ EXPECT_FALSE(match.empty());
+ EXPECT_STREQ(match[0].string, "goodbye");
+}
+
+TEST(TokenDatabase, SingleEntryLookup_LastEntry) {
+ auto match = kBasicDatabase.Find(0xff);
+ ASSERT_EQ(match.size(), 1u);
+ EXPECT_STREQ(match[0].string, ":)");
+ EXPECT_FALSE(match.empty());
+}
+
+TEST(TokenDatabase, SingleEntryLookup_NonPresent) {
+ EXPECT_TRUE(kBasicDatabase.Find(0).empty());
+ EXPECT_TRUE(kBasicDatabase.Find(3).empty());
+ EXPECT_TRUE(kBasicDatabase.Find(10239).empty());
+ EXPECT_TRUE(kBasicDatabase.Find(0xFFFFFFFFu).empty());
+}
+
+TEST(TokenDatabase, SingleEntryLookup_NoMatches) {
+ // Can also create the database at runtime.
+ TokenDatabase tokens = TokenDatabase::Create(kBasicData);
+ const auto match = tokens.Find(42);
+ ASSERT_EQ(match.size(), 0u);
+ EXPECT_TRUE(match.empty());
+
+ for (const auto& entry : match) {
+ FAIL(); // There were no matches, so this code should never execute.
+ static_cast<void>(entry);
+ }
+}
+
+alignas(TokenDatabase::RawEntry) constexpr char kCollisionsData[] =
+ "TOKENS\0\0\x05\0\0\0\0\0\0\0"
+ "\x01\0\0\0date"
+ "\x01\0\0\0date"
+ "\x01\0\0\0date"
+ "\x02\0\0\0date"
+ "\xFF\0\0\0date"
+ "hi!\0goodbye\0:)\0\0";
+
+constexpr TokenDatabase kCollisions = TokenDatabase::Create<kCollisionsData>();
+static_assert(kCollisions.size() == 5u);
+
+TEST(TokenDatabase, MultipleEntriesWithSameToken) {
+ TokenDatabase::Entries match = kCollisions.Find(1);
+
+ EXPECT_EQ(match.begin()->token, 1u);
+ EXPECT_EQ(match.end()->token, 2u);
+ ASSERT_EQ(match.size(), 3u);
+
+ EXPECT_STREQ(match[0].string, "hi!");
+ EXPECT_STREQ(match[1].string, "goodbye");
+ EXPECT_STREQ(match[2].string, ":)");
+
+ for (const auto& entry : match) {
+ EXPECT_EQ(entry.token, 1u);
+ }
+}
+
+TEST(TokenDatabase, Empty) {
+ constexpr TokenDatabase empty_db = TokenDatabase::Create<kEmptyData>();
+ static_assert(empty_db.size() == 0u);
+ static_assert(empty_db.ok());
+
+ EXPECT_TRUE(empty_db.Find(0).empty());
+ EXPECT_TRUE(empty_db.Find(123).empty());
+
+ for (const auto& entry : empty_db) {
+ FAIL(); // The database is empty; this should never execute.
+ static_cast<void>(entry);
+ }
+}
+
+TEST(TokenDatabase, NullDatabase) {
+ constexpr TokenDatabase empty_db;
+
+ static_assert(empty_db.size() == 0u);
+ static_assert(!empty_db.ok());
+ EXPECT_TRUE(empty_db.Find(0).empty());
+}
+
+TEST(TokenDatabase, InvalidData) {
+ constexpr TokenDatabase bad_db = TokenDatabase::Create("TOKENS\0\0");
+
+ static_assert(!bad_db.ok());
+ EXPECT_TRUE(bad_db.Find(0).empty());
+}
+
+TEST(TokenDatabase, FromString) {
+ TokenDatabase bad_db = TokenDatabase::Create(std::string("wow!"));
+
+ EXPECT_FALSE(bad_db.ok());
+}
+
+} // namespace
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/tokenize.cc b/pw_tokenizer/tokenize.cc
new file mode 100644
index 0000000..18bf88d
--- /dev/null
+++ b/pw_tokenizer/tokenize.cc
@@ -0,0 +1,261 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// This file defines the functions that encode tokenized logs at runtime. These
+// are the only pw_tokenizer functions present in a binary that tokenizes
+// strings. All other tokenizing code is resolved at compile time.
+
+#include "pw_tokenizer/tokenize.h"
+
+#include <algorithm>
+#include <array>
+#include <cstdarg>
+#include <cstddef>
+#include <cstring>
+
+#include "pw_varint/varint.h"
+
+namespace pw::tokenizer {
+namespace {
+
+// Store metadata about this compilation's string tokenization in the ELF.
+//
+// The tokenizer metadata will not go into the on-device executable binary code.
+// This metadata will be present in the ELF file's .tokenizer_info section, from
+// which the host-side tooling (Python, Java, etc.) can understand how to decode
+// tokenized strings for the given binary. Only attributes that affect the
+// decoding process are recorded.
+//
+// Tokenizer metadata is stored in an array of key-value pairs. Each Metadata
+// object is 32 bytes: a 24-byte string and an 8-byte value. Metadata structs
+// may be parsed in Python with the struct format '24s<Q'.
+PW_PACKED(struct) Metadata {
+ char name[24]; // name of the metadata field
+ uint64_t value; // value of the field
+};
+
+static_assert(sizeof(Metadata) == 32);
+
+// Store tokenization metadata in its own section.
+constexpr Metadata metadata[] PW_KEEP_IN_SECTION(".tokenzier_info") = {
+ {"hash_length_bytes", PW_TOKENIZER_CFG_HASH_LENGTH},
+ {"sizeof_long", sizeof(long)}, // %l conversion specifier
+ {"sizeof_intmax_t", sizeof(intmax_t)}, // %j conversion specifier
+ {"sizeof_size_t", sizeof(size_t)}, // %z conversion specifier
+ {"sizeof_ptrdiff_t", sizeof(ptrdiff_t)}, // %t conversion specifier
+};
+
+// Declare the types as an enum for convenience.
+enum class ArgType : uint8_t {
+ kInt = PW_TOKENIZER_ARG_TYPE_INT,
+ kInt64 = PW_TOKENIZER_ARG_TYPE_INT64,
+ kDouble = PW_TOKENIZER_ARG_TYPE_DOUBLE,
+ kString = PW_TOKENIZER_ARG_TYPE_STRING,
+};
+
+// Just to be safe, make sure these values are what we expect them to be.
+static_assert(0b00u == static_cast<uint8_t>(ArgType::kInt));
+static_assert(0b01u == static_cast<uint8_t>(ArgType::kInt64));
+static_assert(0b10u == static_cast<uint8_t>(ArgType::kDouble));
+static_assert(0b11u == static_cast<uint8_t>(ArgType::kString));
+
+// Buffer for encoding a tokenized string and arguments.
+struct EncodedMessage {
+ pw_TokenizerStringToken token;
+ std::array<uint8_t, PW_TOKENIZER_CFG_ENCODING_BUFFER_SIZE_BYTES> args;
+};
+
+static_assert(offsetof(EncodedMessage, args) == sizeof(EncodedMessage::token),
+ "EncodedMessage should not have padding bytes between members");
+
+size_t EncodeInt(int value, const span<uint8_t>& output) {
+ return varint::Encode(value, pw::as_writable_bytes(output));
+}
+
+size_t EncodeInt64(int64_t value, const span<uint8_t>& output) {
+ return varint::Encode(value, pw::as_writable_bytes(output));
+}
+
+size_t EncodeFloat(float value, const span<uint8_t>& output) {
+ if (output.size() < sizeof(value)) {
+ return 0;
+ }
+ std::memcpy(output.data(), &value, sizeof(value));
+ return sizeof(value);
+}
+
+size_t EncodeString(const char* string, const span<uint8_t>& output) {
+ // The top bit of the status byte indicates if the string was truncated.
+ static constexpr size_t kMaxStringLength = 0x7Fu;
+
+ if (output.empty()) { // At least one byte is needed for the status/size.
+ return 0;
+ }
+
+ if (string == nullptr) {
+ string = "NULL";
+ }
+
+ // Subtract 1 to save room for the status byte.
+ const size_t max_bytes = std::min(output.size(), kMaxStringLength) - 1;
+
+ // Scan the string to find out how many bytes to copy.
+ size_t bytes_to_copy = 0;
+ uint8_t overflow_bit = 0;
+
+ while (string[bytes_to_copy] != '\0') {
+ if (bytes_to_copy == max_bytes) {
+ overflow_bit = '\x80';
+ break;
+ }
+ bytes_to_copy += 1;
+ }
+
+ output[0] = bytes_to_copy | overflow_bit;
+ std::memcpy(output.data() + 1, string, bytes_to_copy);
+
+ return bytes_to_copy + 1; // include the status byte in the total
+}
+
+size_t EncodeArgs(pw_TokenizerArgTypes types,
+ va_list args,
+ span<uint8_t> output) {
+ size_t arg_count = types & PW_TOKENIZER_TYPE_COUNT_MASK;
+ types >>= PW_TOKENIZER_TYPE_COUNT_SIZE_BITS;
+
+ size_t encoded_bytes = 0;
+ while (arg_count != 0u) {
+ // How many bytes were encoded; 0 indicates that there wasn't enough space.
+ size_t argument_bytes = 0;
+
+ switch (static_cast<ArgType>(types & 0b11u)) {
+ case ArgType::kInt:
+ argument_bytes = EncodeInt(va_arg(args, int), output);
+ break;
+ case ArgType::kInt64:
+ argument_bytes = EncodeInt64(va_arg(args, int64_t), output);
+ break;
+ case ArgType::kDouble:
+ argument_bytes =
+ EncodeFloat(static_cast<float>(va_arg(args, double)), output);
+ break;
+ case ArgType::kString:
+ argument_bytes = EncodeString(va_arg(args, const char*), output);
+ break;
+ }
+
+ // If zero bytes were encoded, the encoding buffer is full.
+ if (argument_bytes == 0u) {
+ break;
+ }
+
+ output = output.subspan(argument_bytes);
+ encoded_bytes += argument_bytes;
+
+ arg_count -= 1;
+ types >>= 2; // each argument type is encoded in two bits
+ }
+
+ return encoded_bytes;
+}
+
+} // namespace
+
+extern "C" {
+
+void pw_TokenizeToBuffer(void* buffer,
+ size_t* buffer_size_bytes,
+ pw_TokenizerStringToken token,
+ pw_TokenizerArgTypes types,
+ ...) {
+ if (*buffer_size_bytes < sizeof(token)) {
+ *buffer_size_bytes = 0;
+ return;
+ }
+
+ std::memcpy(buffer, &token, sizeof(token));
+
+ va_list args;
+ va_start(args, types);
+ const size_t encoded_bytes =
+ EncodeArgs(types,
+ args,
+ span(static_cast<uint8_t*>(buffer) + sizeof(token),
+ *buffer_size_bytes - sizeof(token)));
+ va_end(args);
+
+ *buffer_size_bytes = sizeof(token) + encoded_bytes;
+}
+
+void pw_TokenizeToCallback(void (*callback)(const uint8_t* encoded_message,
+ size_t size_bytes),
+ pw_TokenizerStringToken token,
+ pw_TokenizerArgTypes types,
+ ...) {
+ EncodedMessage encoded;
+ encoded.token = token;
+
+ va_list args;
+ va_start(args, types);
+ const size_t encoded_bytes = EncodeArgs(types, args, encoded.args);
+ va_end(args);
+
+ callback(reinterpret_cast<const uint8_t*>(&encoded),
+ sizeof(encoded.token) + encoded_bytes);
+}
+
+#if PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+void pw_TokenizeToGlobalHandler(pw_TokenizerStringToken token,
+ pw_TokenizerArgTypes types,
+ ...) {
+ EncodedMessage encoded;
+ encoded.token = token;
+
+ va_list args;
+ va_start(args, types);
+ const size_t encoded_bytes = EncodeArgs(types, args, encoded.args);
+ va_end(args);
+
+ pw_TokenizerHandleEncodedMessage(reinterpret_cast<const uint8_t*>(&encoded),
+ sizeof(encoded.token) + encoded_bytes);
+}
+
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+#if PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+
+void pw_TokenizeToGlobalHandlerWithPayload(const pw_TokenizerPayload payload,
+ pw_TokenizerStringToken token,
+ pw_TokenizerArgTypes types,
+ ...) {
+ EncodedMessage encoded;
+ encoded.token = token;
+
+ va_list args;
+ va_start(args, types);
+ const size_t encoded_bytes = EncodeArgs(types, args, encoded.args);
+ va_end(args);
+
+ pw_TokenizerHandleEncodedMessageWithPayload(
+ payload,
+ reinterpret_cast<const uint8_t*>(&encoded),
+ sizeof(encoded.token) + encoded_bytes);
+}
+
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+
+} // extern "C"
+
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/tokenize_test.c b/pw_tokenizer/tokenize_test.c
new file mode 100644
index 0000000..1574451
--- /dev/null
+++ b/pw_tokenizer/tokenize_test.c
@@ -0,0 +1,124 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+// This function tests the C implementation of tokenization API. These functions
+// are called from the main C++ test file.
+
+#include "pw_tokenizer_private/tokenize_test.h"
+
+#include "pw_tokenizer/tokenize.h"
+
+#ifdef __cplusplus
+#error "This is a test of C code and must be compiled as C, not C++."
+#endif // __cplusplus
+
+void pw_TokenizeToBufferTest_StringShortFloat(void* buffer,
+ size_t* buffer_size) {
+ char str[] = "1";
+ PW_TOKENIZE_TO_BUFFER(
+ buffer, buffer_size, TEST_FORMAT_STRING_SHORT_FLOAT, str, (short)-2, 3.0);
+}
+
+// This test invokes the tokenization API with a variety of types. To simplify
+// validating the encoded data, numbers that are sequential when zig-zag encoded
+// are used as arguments.
+void pw_TokenizeToBufferTest_SequentialZigZag(void* buffer,
+ size_t* buffer_size) {
+ PW_TOKENIZE_TO_BUFFER(buffer,
+ buffer_size,
+ TEST_FORMAT_SEQUENTIAL_ZIG_ZAG,
+ 0u,
+ -1,
+ 1u,
+ (unsigned)-2,
+ (unsigned short)2u,
+ (signed char)-3,
+ 3,
+ -4l,
+ 4ul,
+ -5ll,
+ 5ull,
+ (signed char)-6,
+ (char)6,
+ (signed char)-7);
+}
+
+void pw_TokenizeToCallbackTest_SequentialZigZag(
+ void (*callback)(const uint8_t* buffer, size_t size)) {
+ PW_TOKENIZE_TO_CALLBACK(callback,
+ TEST_FORMAT_SEQUENTIAL_ZIG_ZAG,
+ 0u,
+ -1,
+ 1u,
+ (unsigned)-2,
+ (unsigned short)2u,
+ (signed char)-3,
+ 3,
+ -4l,
+ 4ul,
+ -5ll,
+ 5ull,
+ (signed char)-6,
+ (char)6,
+ (signed char)-7);
+}
+
+#if PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+void pw_TokenizeToGlobalHandlerTest_SequentialZigZag(void) {
+ PW_TOKENIZE_TO_GLOBAL_HANDLER(TEST_FORMAT_SEQUENTIAL_ZIG_ZAG,
+ 0u,
+ -1,
+ 1u,
+ (unsigned)-2,
+ (unsigned short)2u,
+ (signed char)-3,
+ 3,
+ -4l,
+ 4ul,
+ -5ll,
+ 5ull,
+ (signed char)-6,
+ (char)6,
+ (signed char)-7);
+}
+
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+#if PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+
+void pw_TokenizeToGlobalHandlerWithPayloadTest_SequentialZigZag(void) {
+ PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD((pw_TokenizerPayload)600613,
+ TEST_FORMAT_SEQUENTIAL_ZIG_ZAG,
+ 0u,
+ -1,
+ 1u,
+ (unsigned)-2,
+ (unsigned short)2u,
+ (signed char)-3,
+ 3,
+ -4l,
+ 4ul,
+ -5ll,
+ 5ull,
+ (signed char)-6,
+ (char)6,
+ (signed char)-7);
+}
+
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+
+void pw_TokenizeToBufferTest_Requires8(void* buffer, size_t* buffer_size) {
+ PW_TOKENIZE_TO_BUFFER(buffer, buffer_size, TEST_FORMAT_REQUIRES_8, "hi", -7);
+}
diff --git a/pw_tokenizer/tokenize_test.cc b/pw_tokenizer/tokenize_test.cc
new file mode 100644
index 0000000..73ffaab
--- /dev/null
+++ b/pw_tokenizer/tokenize_test.cc
@@ -0,0 +1,542 @@
+// Copyright 2020 The Pigweed Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"); you may not
+// use this file except in compliance with the License. You may obtain a copy of
+// the License at
+//
+// https://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+// License for the specific language governing permissions and limitations under
+// the License.
+
+#include "pw_tokenizer/tokenize.h"
+
+#include <cinttypes>
+#include <cstdint>
+#include <cstring>
+#include <iterator>
+
+#include "gtest/gtest.h"
+#include "pw_tokenizer/pw_tokenizer_65599_fixed_length_hash.h"
+#include "pw_tokenizer_private/tokenize_test.h"
+#include "pw_varint/varint.h"
+
+namespace pw::tokenizer {
+namespace {
+
+// The hash to use for this test. This makes sure the strings are shorter than
+// the configured max length to ensure this test works with any reasonable
+// configuration.
+template <size_t kSize>
+constexpr uint32_t TestHash(const char (&string)[kSize]) {
+ constexpr unsigned kTestHashLength = 48;
+ static_assert(kTestHashLength <= PW_TOKENIZER_CFG_HASH_LENGTH);
+ static_assert(kSize <= kTestHashLength + 1);
+ return PwTokenizer65599FixedLengthHash(std::string_view(string, kSize - 1),
+ kTestHashLength);
+}
+
+// Constructs an array with the hashed string followed by the provided bytes.
+template <uint8_t... kData, size_t kSize>
+constexpr auto ExpectedData(const char (&format)[kSize]) {
+ const uint32_t value = TestHash(format);
+ return std::array<uint8_t, sizeof(uint32_t) + sizeof...(kData)>{
+ static_cast<uint8_t>(value & 0xff),
+ static_cast<uint8_t>(value >> 8 & 0xff),
+ static_cast<uint8_t>(value >> 16 & 0xff),
+ static_cast<uint8_t>(value >> 24 & 0xff),
+ kData...};
+}
+
+TEST(TokenizeStringLiteral, EmptyString_IsZero) {
+ constexpr pw_TokenizerStringToken token = PW_TOKENIZE_STRING("");
+ EXPECT_EQ(0u, token);
+}
+
+TEST(TokenizeStringLiteral, String_MatchesHash) {
+ constexpr uint32_t token = PW_TOKENIZE_STRING("[:-)");
+ EXPECT_EQ(TestHash("[:-)"), token);
+}
+
+constexpr uint32_t kGlobalToken = PW_TOKENIZE_STRING(">:-[]");
+
+TEST(TokenizeStringLiteral, GlobalVariable_MatchesHash) {
+ EXPECT_EQ(TestHash(">:-[]"), kGlobalToken);
+}
+
+class TokenizeToBuffer : public ::testing::Test {
+ public:
+ TokenizeToBuffer() : buffer_{} {}
+
+ protected:
+ uint8_t buffer_[64];
+};
+
+TEST_F(TokenizeToBuffer, Integer64) {
+ size_t message_size = 14;
+ PW_TOKENIZE_TO_BUFFER(
+ buffer_,
+ &message_size,
+ "%" PRIu64,
+ static_cast<uint64_t>(0x55555555'55555555ull)); // 0xAAAAAAAA'AAAAAAAA
+
+ // Pattern becomes 10101010'11010101'10101010 ...
+ constexpr std::array<uint8_t, 14> expected =
+ ExpectedData<0xAA, 0xD5, 0xAA, 0xD5, 0xAA, 0xD5, 0xAA, 0xD5, 0xAA, 0x01>(
+ "%" PRIu64);
+ ASSERT_EQ(expected.size(), message_size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, Integer64Overflow) {
+ size_t message_size;
+
+ for (size_t size = 4; size < 20; ++size) {
+ message_size = size;
+
+ PW_TOKENIZE_TO_BUFFER(
+ buffer_,
+ &message_size,
+ "%" PRIx64,
+ static_cast<uint64_t>(std::numeric_limits<int64_t>::min()));
+
+ if (size < 14) {
+ constexpr std::array<uint8_t, 4> empty = ExpectedData("%" PRIx64);
+ ASSERT_EQ(sizeof(uint32_t), message_size);
+ EXPECT_EQ(std::memcmp(empty.data(), &buffer_, empty.size()), 0);
+
+ // Make sure nothing was written past the end of the buffer.
+ EXPECT_TRUE(std::all_of(&buffer_[size], std::end(buffer_), [](uint8_t v) {
+ return v == '\0';
+ }));
+ } else {
+ constexpr std::array<uint8_t, 14> expected =
+ ExpectedData<0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0xff,
+ 0x01>("%" PRIx64);
+ ASSERT_EQ(expected.size(), message_size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+ }
+ }
+}
+
+TEST_F(TokenizeToBuffer, IntegerNegative) {
+ size_t message_size = 9;
+ PW_TOKENIZE_TO_BUFFER(
+ buffer_, &message_size, "%" PRId32, std::numeric_limits<int32_t>::min());
+
+ // 0x8000'0000 -zig-zag-> 0xff'ff'ff'ff'0f
+ constexpr std::array<uint8_t, 9> expected =
+ ExpectedData<0xff, 0xff, 0xff, 0xff, 0x0f>("%" PRId32);
+ ASSERT_EQ(expected.size(), message_size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, IntegerMin) {
+ size_t message_size = 9;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "%d", -1);
+
+ constexpr std::array<uint8_t, 5> expected = ExpectedData<0x01>("%d");
+ ASSERT_EQ(expected.size(), message_size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, IntegerDoesntFit) {
+ size_t message_size = 8;
+ PW_TOKENIZE_TO_BUFFER(
+ buffer_, &message_size, "%" PRId32, std::numeric_limits<int32_t>::min());
+
+ constexpr std::array<uint8_t, 4> expected = ExpectedData<>("%" PRId32);
+ ASSERT_EQ(expected.size(), message_size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, String) {
+ size_t message_size = sizeof(buffer_);
+
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "The answer is: %s", "5432!");
+ constexpr std::array<uint8_t, 10> expected =
+ ExpectedData<5, '5', '4', '3', '2', '!'>("The answer is: %s");
+
+ ASSERT_EQ(expected.size(), message_size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, String_BufferTooSmall_TruncatesAndSetsTopStatusBit) {
+ size_t message_size = 8;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "The answer is: %s", "5432!");
+
+ constexpr std::array<uint8_t, 8> truncated_1 =
+ ExpectedData<0x83, '5', '4', '3'>("The answer is: %s");
+
+ ASSERT_EQ(truncated_1.size(), message_size);
+ EXPECT_EQ(std::memcmp(truncated_1.data(), buffer_, truncated_1.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, String_TwoBytesLeft_TruncatesToOneCharacter) {
+ size_t message_size = 6;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "The answer is: %s", "5432!");
+
+ constexpr std::array<uint8_t, 6> truncated_2 =
+ ExpectedData<0x81, '5'>("The answer is: %s");
+
+ ASSERT_EQ(truncated_2.size(), message_size);
+ EXPECT_EQ(std::memcmp(truncated_2.data(), buffer_, truncated_2.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, String_OneByteLeft_OnlyWritesTruncatedStatusByte) {
+ size_t message_size = 5;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "The answer is: %s", "5432!");
+
+ std::array<uint8_t, 5> result = ExpectedData<0x80>("The answer is: %s");
+ ASSERT_EQ(result.size(), message_size);
+ EXPECT_EQ(std::memcmp(result.data(), buffer_, result.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, EmptyString_OneByteLeft_EncodesCorrectly) {
+ size_t message_size = 5;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "The answer is: %s", "");
+
+ std::array<uint8_t, 5> result = ExpectedData<0>("The answer is: %s");
+ ASSERT_EQ(result.size(), message_size);
+ EXPECT_EQ(std::memcmp(result.data(), buffer_, result.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, String_ZeroBytesLeft_WritesNothing) {
+ size_t message_size = 4;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "The answer is: %s", "5432!");
+
+ constexpr std::array<uint8_t, 4> empty = ExpectedData<>("The answer is: %s");
+ ASSERT_EQ(empty.size(), message_size);
+ EXPECT_EQ(std::memcmp(empty.data(), buffer_, empty.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, NullptrString_EncodesNull) {
+ char* string = nullptr;
+ size_t message_size = 9;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "The answer is: %s", string);
+
+ std::array<uint8_t, 9> result =
+ ExpectedData<4, 'N', 'U', 'L', 'L'>("The answer is: %s");
+ ASSERT_EQ(result.size(), message_size);
+ EXPECT_EQ(std::memcmp(result.data(), buffer_, result.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, NullptrString_BufferTooSmall_EncodesTruncatedNull) {
+ char* string = nullptr;
+ size_t message_size = 6;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "The answer is: %s", string);
+
+ std::array<uint8_t, 6> result = ExpectedData<0x81, 'N'>("The answer is: %s");
+ ASSERT_EQ(result.size(), message_size);
+ EXPECT_EQ(std::memcmp(result.data(), buffer_, result.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, TruncateArgs) {
+ // Args that can't fit are dropped completely
+ size_t message_size = 6;
+ PW_TOKENIZE_TO_BUFFER(buffer_,
+ &message_size,
+ "%u %d",
+ static_cast<uint8_t>(0b0010'1010u),
+ 0xffffff);
+
+ constexpr std::array<uint8_t, 5> expected =
+ ExpectedData<0b0101'0100u>("%u %d");
+ ASSERT_EQ(expected.size(), message_size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, NoRoomForToken) {
+ // Nothing is written if there isn't room for the token.
+ std::memset(buffer_, '$', sizeof(buffer_));
+ auto is_untouched = [](uint8_t v) { return v == '$'; };
+
+ size_t message_size = 3;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "The answer: \"%s\"", "5432!");
+ EXPECT_EQ(0u, message_size);
+ EXPECT_TRUE(std::all_of(buffer_, std::end(buffer_), is_untouched));
+
+ message_size = 2;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "Jello, world!");
+ EXPECT_EQ(0u, message_size);
+ EXPECT_TRUE(std::all_of(buffer_, std::end(buffer_), is_untouched));
+
+ message_size = 1;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "Jello!");
+ EXPECT_EQ(0u, message_size);
+ EXPECT_TRUE(std::all_of(buffer_, std::end(buffer_), is_untouched));
+
+ message_size = 0;
+ PW_TOKENIZE_TO_BUFFER(buffer_, &message_size, "Jello?");
+ EXPECT_EQ(0u, message_size);
+ EXPECT_TRUE(std::all_of(buffer_, std::end(buffer_), is_untouched));
+}
+
+TEST_F(TokenizeToBuffer, C_StringShortFloat) {
+ size_t size = sizeof(buffer_);
+ pw_TokenizeToBufferTest_StringShortFloat(buffer_, &size);
+ constexpr std::array<uint8_t, 11> expected = // clang-format off
+ ExpectedData<1, '1', // string '1'
+ 3, // -2 (zig-zag encoded)
+ 0x00, 0x00, 0x40, 0x40 // 3.0 in floating point
+ >(TEST_FORMAT_STRING_SHORT_FLOAT);
+ ASSERT_EQ(expected.size(), size); // clang-format on
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, C_SequentialZigZag) {
+ size_t size = sizeof(buffer_);
+ pw_TokenizeToBufferTest_SequentialZigZag(buffer_, &size);
+ constexpr std::array<uint8_t, 18> expected =
+ ExpectedData<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13>(
+ TEST_FORMAT_SEQUENTIAL_ZIG_ZAG);
+
+ ASSERT_EQ(expected.size(), size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToBuffer, C_Overflow) {
+ std::memset(buffer_, '$', sizeof(buffer_));
+
+ {
+ size_t size = 7;
+ pw_TokenizeToBufferTest_Requires8(buffer_, &size);
+ constexpr std::array<uint8_t, 7> expected =
+ ExpectedData<2, 'h', 'i'>(TEST_FORMAT_REQUIRES_8);
+ ASSERT_EQ(expected.size(), size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+ EXPECT_EQ(buffer_[7], '$');
+ }
+
+ {
+ size_t size = 8;
+ pw_TokenizeToBufferTest_Requires8(buffer_, &size);
+ constexpr std::array<uint8_t, 8> expected =
+ ExpectedData<2, 'h', 'i', 13>(TEST_FORMAT_REQUIRES_8);
+ ASSERT_EQ(expected.size(), size);
+ EXPECT_EQ(std::memcmp(expected.data(), buffer_, expected.size()), 0);
+ EXPECT_EQ(buffer_[8], '$');
+ }
+}
+
+// Test fixture for callback and global handler. Both of these need a global
+// message buffer. To keep the message buffers separate, template this on the
+// derived class type.
+template <typename Impl>
+class GlobalMessage : public ::testing::Test {
+ public:
+ static void SetMessage(const uint8_t* message, size_t size) {
+ ASSERT_LE(size, sizeof(message_));
+ std::memcpy(message_, message, size);
+ message_size_bytes_ = size;
+ }
+
+ protected:
+ GlobalMessage() {
+ std::memset(message_, 0, sizeof(message_));
+ message_size_bytes_ = 0;
+ }
+
+ static uint8_t message_[256];
+ static size_t message_size_bytes_;
+};
+
+template <typename Impl>
+uint8_t GlobalMessage<Impl>::message_[256] = {};
+template <typename Impl>
+size_t GlobalMessage<Impl>::message_size_bytes_ = 0;
+
+class TokenizeToCallback : public GlobalMessage<TokenizeToCallback> {};
+
+TEST_F(TokenizeToCallback, Variety) {
+ PW_TOKENIZE_TO_CALLBACK(
+ SetMessage, "%s there are %x (%.2f) of them%c", "Now", 2u, 2.0f, '.');
+ const auto expected = // clang-format off
+ ExpectedData<3, 'N', 'o', 'w', // string "Now"
+ 0x04, // unsigned 2 (zig-zag encoded)
+ 0x00, 0x00, 0x00, 0x40, // float 2.0
+ 0x5C // char '.' (0x2E, zig-zag encoded)
+ >("%s there are %x (%.2f) of them%c");
+ // clang-format on
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToCallback, Strings) {
+ PW_TOKENIZE_TO_CALLBACK(SetMessage, "The answer is: %s", "5432!");
+ constexpr std::array<uint8_t, 10> expected =
+ ExpectedData<5, '5', '4', '3', '2', '!'>("The answer is: %s");
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToCallback, C_SequentialZigZag) {
+ pw_TokenizeToCallbackTest_SequentialZigZag(SetMessage);
+
+ constexpr std::array<uint8_t, 18> expected =
+ ExpectedData<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13>(
+ TEST_FORMAT_SEQUENTIAL_ZIG_ZAG);
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+}
+
+#if PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+class TokenizeToGlobalHandler : public GlobalMessage<TokenizeToGlobalHandler> {
+};
+
+TEST_F(TokenizeToGlobalHandler, Variety) {
+ PW_TOKENIZE_TO_GLOBAL_HANDLER("%x%lld%1.2f%s", 0, 0ll, -0.0, "");
+ const auto expected =
+ ExpectedData<0, 0, 0x00, 0x00, 0x00, 0x80, 0>("%x%lld%1.2f%s");
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToGlobalHandler, Strings) {
+ PW_TOKENIZE_TO_GLOBAL_HANDLER("The answer is: %s", "5432!");
+ constexpr std::array<uint8_t, 10> expected =
+ ExpectedData<5, '5', '4', '3', '2', '!'>("The answer is: %s");
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+}
+
+TEST_F(TokenizeToGlobalHandler, C_SequentialZigZag) {
+ pw_TokenizeToGlobalHandlerTest_SequentialZigZag();
+
+ constexpr std::array<uint8_t, 18> expected =
+ ExpectedData<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13>(
+ TEST_FORMAT_SEQUENTIAL_ZIG_ZAG);
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+}
+
+extern "C" void pw_TokenizerHandleEncodedMessage(const uint8_t* encoded_message,
+ size_t size_bytes) {
+ TokenizeToGlobalHandler::SetMessage(encoded_message, size_bytes);
+}
+
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER
+
+#if PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+
+class TokenizeToGlobalHandlerWithPayload
+ : public GlobalMessage<TokenizeToGlobalHandlerWithPayload> {
+ public:
+ static void SetPayload(pw_TokenizerPayload payload) {
+ payload_ = reinterpret_cast<intptr_t>(payload);
+ }
+
+ protected:
+ TokenizeToGlobalHandlerWithPayload() { payload_ = {}; }
+
+ static intptr_t payload_;
+};
+
+intptr_t TokenizeToGlobalHandlerWithPayload::payload_;
+
+TEST_F(TokenizeToGlobalHandlerWithPayload, Variety) {
+ ASSERT_NE(payload_, 123);
+
+ const auto expected =
+ ExpectedData<0, 0, 0x00, 0x00, 0x00, 0x80, 0>("%x%lld%1.2f%s");
+
+ PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD(
+ reinterpret_cast<pw_TokenizerPayload>(123),
+ "%x%lld%1.2f%s",
+ 0,
+ 0ll,
+ -0.0,
+ "");
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+ EXPECT_EQ(payload_, 123);
+
+ PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD(
+ reinterpret_cast<pw_TokenizerPayload>(-543),
+ "%x%lld%1.2f%s",
+ 0,
+ 0ll,
+ -0.0,
+ "");
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+ EXPECT_EQ(payload_, -543);
+}
+
+TEST_F(TokenizeToGlobalHandlerWithPayload, Strings) {
+ constexpr std::array<uint8_t, 10> expected =
+ ExpectedData<5, '5', '4', '3', '2', '!'>("The answer is: %s");
+
+ PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD(
+ reinterpret_cast<pw_TokenizerPayload>(5432),
+ "The answer is: %s",
+ "5432!");
+
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+ EXPECT_EQ(payload_, 5432);
+
+ PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD({}, "The answer is: %s", "5432!");
+
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+ EXPECT_EQ(payload_, 0);
+}
+
+struct Foo {
+ unsigned char a;
+ bool b;
+};
+
+TEST_F(TokenizeToGlobalHandlerWithPayload, PointerToStack) {
+ Foo foo{254u, true};
+
+ PW_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD(
+ reinterpret_cast<pw_TokenizerPayload>(&foo), "Boring!");
+
+ constexpr auto expected = ExpectedData("Boring!");
+ static_assert(expected.size() == 4);
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+
+ Foo* payload_foo = reinterpret_cast<Foo*>(payload_);
+ ASSERT_EQ(&foo, payload_foo);
+ EXPECT_EQ(payload_foo->a, 254u);
+ EXPECT_TRUE(payload_foo->b);
+}
+
+TEST_F(TokenizeToGlobalHandlerWithPayload, C_SequentialZigZag) {
+ pw_TokenizeToGlobalHandlerWithPayloadTest_SequentialZigZag();
+
+ constexpr std::array<uint8_t, 18> expected =
+ ExpectedData<0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13>(
+ TEST_FORMAT_SEQUENTIAL_ZIG_ZAG);
+ ASSERT_EQ(expected.size(), message_size_bytes_);
+ EXPECT_EQ(std::memcmp(expected.data(), message_, expected.size()), 0);
+ EXPECT_EQ(payload_, 600613);
+}
+
+extern "C" void pw_TokenizerHandleEncodedMessageWithPayload(
+ pw_TokenizerPayload payload,
+ const uint8_t* encoded_message,
+ size_t size_bytes) {
+ TokenizeToGlobalHandlerWithPayload::SetMessage(encoded_message, size_bytes);
+ TokenizeToGlobalHandlerWithPayload::SetPayload(payload);
+}
+
+#endif // PW_TOKENIZER_CFG_ENABLE_TOKENIZE_TO_GLOBAL_HANDLER_WITH_PAYLOAD
+
+} // namespace
+} // namespace pw::tokenizer
diff --git a/pw_tokenizer/tokenizer_linker_sections.ld b/pw_tokenizer/tokenizer_linker_sections.ld
new file mode 100644
index 0000000..db08481
--- /dev/null
+++ b/pw_tokenizer/tokenizer_linker_sections.ld
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2020 The Pigweed Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ *
+ * This linker script snippet declares the sections needed for string
+ * tokenization.
+ *
+ * This file may be directly included in a linker script with an include
+ * directive. For example,
+ *
+ * INCLUDE path/to/modules/pw_tokenizer/tokenizer_linker_sections.ld
+ *
+ * SECTIONS
+ * {
+ * (your existing linker sections)
+ * }
+ */
+
+SECTIONS
+{
+ /*
+ * All tokenized strings are stored in this section. Since the section has
+ * type INFO, it is excluded from the final binary.
+ *
+ * In the compiled code, format string literals are replaced by a hash of the
+ * string contents and a compact argument list encoded in a uint32_t. The
+ * compiled code contains no references to the tokenized strings in this
+ * section.
+ *
+ * The section contents are declared with KEEP so that they are not removed
+ * from the ELF. These are never emitted in the final binary or loaded into
+ * memory.
+ */
+ .tokenized 0x00000000 (INFO) :
+ {
+ KEEP(*(.tokenized))
+ KEEP(*(.tokenized.*))
+ }
+
+ /*
+ * This section stores metadata that may be used during tokenized string
+ * decoding. This metadata describes properties that may affect how the
+ * tokenized string is encoded or decoded -- the maximum length of the hash
+ * function and the sizes of certain integer types.
+ *
+ * Metadata is declared as key-value pairs. See the metadata variable in
+ * tokenize.cc for further details.
+ */
+ .tokenizer_info 0x00000000 (INFO) :
+ {
+ KEEP(*(.tokenizer_info))
+ }
+}
