pw_log: Add utility to encode tokenized message Adds EncodeTokenizedMessage to support easy translation between pw_log_tokenized data and the log protobuf format. Change-Id: I4667eb9de92e09a604541e0a667fd8cab2cdde5b Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/47881 Pigweed-Auto-Submit: Prashanth Swaminathan <prashanthsw@google.com> Commit-Queue: Auto-Submit <auto-submit@pigweed.google.com.iam.gserviceaccount.com> Reviewed-by: Keir Mierle <keir@google.com> Reviewed-by: Wyatt Hepler <hepler@google.com>
diff --git a/pw_log/BUILD b/pw_log/BUILD index d76a685..9d58517 100644 --- a/pw_log/BUILD +++ b/pw_log/BUILD
@@ -47,6 +47,22 @@ ) pw_cc_library( + name = "proto_utils", + srcs = [ + "proto_utils.cc", + ], + hdrs = [ + "public/pw_log/proto_utils.h", + ], + deps = [ + ":facade", + "//pw_bytes", + "//pw_log_tokenized", + "//pw_result", + ], +) + +pw_cc_library( name = "backend_multiplexer", visibility = ["@pigweed_config//:__pkg__"], deps = ["//pw_log_basic"], @@ -65,3 +81,17 @@ "//pw_unit_test", ], ) + +pw_cc_test( + name = "proto_utils_test", + srcs = [ + "proto_utils_test.cc", + ], + deps = [ + ":facade", + ":proto_utils", + "//pw_protobuf", + "//pw_preprocessor", + "//pw_unit_test", + ], +)
diff --git a/pw_log/BUILD.gn b/pw_log/BUILD.gn index 25b5d0d..d639762 100644 --- a/pw_log/BUILD.gn +++ b/pw_log/BUILD.gn
@@ -15,6 +15,7 @@ import("//build_overrides/pigweed.gni") import("$dir_pw_build/facade.gni") +import("$dir_pw_chrono/backend.gni") import("$dir_pw_docgen/docs.gni") import("$dir_pw_log/backend.gni") import("$dir_pw_protobuf_compiler/proto.gni") @@ -38,6 +39,19 @@ require_link_deps = [ ":impl" ] } +pw_source_set("proto_utils") { + public_configs = [ ":default_config" ] + public = [ "public/pw_log/proto_utils.h" ] + public_deps = [ + ":pw_log.facade", + "$dir_pw_bytes", + "$dir_pw_log_tokenized", + "$dir_pw_result", + ] + deps = [ "$dir_pw_log:protos.pwpb" ] + sources = [ "proto_utils.cc" ] +} + # pw_log is low-level and ubiquitous. Because of this, it can often cause # circular dependencies. This target collects dependencies from the backend that # cannot be used because they would cause circular deps. @@ -58,7 +72,10 @@ } pw_test_group("tests") { - tests = [ ":basic_log_test" ] + tests = [ + ":basic_log_test", + ":proto_utils_test", + ] } pw_test("basic_log_test") { @@ -75,6 +92,17 @@ ] } +pw_test("proto_utils_test") { + enable_if = pw_log_BACKEND != "" + deps = [ + ":proto_utils", + ":pw_log.facade", + "$dir_pw_preprocessor", + "$dir_pw_protobuf", + ] + sources = [ "proto_utils_test.cc" ] +} + pw_proto_library("protos") { sources = [ "log.proto" ] prefix = "pw_log/proto"
diff --git a/pw_log/docs.rst b/pw_log/docs.rst index 9816681..d33240f 100644 --- a/pw_log/docs.rst +++ b/pw_log/docs.rst
@@ -10,9 +10,9 @@ 1. The facade (this module) which is only a macro interface layer 2. The backend, provided elsewhere, that implements the low level logging -``pw_log`` also defines a logging protobuf and RPC service for efficiently -storing and transmitting log messages. See :ref:`module-pw_log-protobuf` for -details. +``pw_log`` also defines a logging protobuf, helper utilities, and an RPC +service for efficiently storing and transmitting log messages. See +:ref:`module-pw_log-protobuf` for details. .. toctree:: :hidden:
diff --git a/pw_log/proto_utils.cc b/pw_log/proto_utils.cc new file mode 100644 index 0000000..002dd4b --- /dev/null +++ b/pw_log/proto_utils.cc
@@ -0,0 +1,44 @@ +// Copyright 2021 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_log/proto_utils.h" + +#include "pw_log/levels.h" +#include "pw_log/proto/log.pwpb.h" +#include "pw_log_tokenized/metadata.h" +#include "pw_protobuf/wire_format.h" + +namespace pw::log { + +Result<ConstByteSpan> EncodeTokenizedLog(pw::log_tokenized::Metadata metadata, + ConstByteSpan tokenized_data, + int64_t ticks_since_epoch, + ByteSpan encode_buffer) { + // Encode message to the LogEntry protobuf. + LogEntry::RamEncoder encoder(encode_buffer); + + encoder.WriteMessage(tokenized_data); + encoder.WriteLineLevel( + (metadata.level() & PW_LOG_LEVEL_BITMASK) | + ((metadata.line_number() << PW_LOG_LEVEL_BITS) & ~PW_LOG_LEVEL_BITMASK)); + if (metadata.flags() != 0) { + encoder.WriteFlags(metadata.flags()); + } + encoder.WriteTimestamp(ticks_since_epoch); + + PW_TRY(encoder.status()); + return ConstByteSpan(encoder); +} + +} // namespace pw::log
diff --git a/pw_log/proto_utils_test.cc b/pw_log/proto_utils_test.cc new file mode 100644 index 0000000..f3ae454 --- /dev/null +++ b/pw_log/proto_utils_test.cc
@@ -0,0 +1,113 @@ +// Copyright 2021 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. + +#include "pw_log/proto_utils.h" + +#include "gtest/gtest.h" +#include "pw_protobuf/decoder.h" + +namespace pw::log { + +void VerifyLogEntry(pw::protobuf::Decoder& entry_decoder, + pw::log_tokenized::Metadata expected_metadata, + ConstByteSpan expected_tokenized_data, + const int64_t expected_timestamp) { + ConstByteSpan tokenized_data; + EXPECT_TRUE(entry_decoder.Next().ok()); // message [tokenized] + EXPECT_EQ(1U, entry_decoder.FieldNumber()); + EXPECT_TRUE(entry_decoder.ReadBytes(&tokenized_data).ok()); + EXPECT_TRUE(std::memcmp(tokenized_data.begin(), + expected_tokenized_data.begin(), + expected_tokenized_data.size()) == 0); + + uint32_t line_level; + EXPECT_TRUE(entry_decoder.Next().ok()); // line_level + EXPECT_EQ(2U, entry_decoder.FieldNumber()); + EXPECT_TRUE(entry_decoder.ReadUint32(&line_level).ok()); + EXPECT_EQ(expected_metadata.level(), line_level & PW_LOG_LEVEL_BITMASK); + EXPECT_EQ(expected_metadata.line_number(), + (line_level & ~PW_LOG_LEVEL_BITMASK) >> PW_LOG_LEVEL_BITS); + + if (expected_metadata.flags() != 0) { + uint32_t flags; + EXPECT_TRUE(entry_decoder.Next().ok()); // flags + EXPECT_EQ(3U, entry_decoder.FieldNumber()); + EXPECT_TRUE(entry_decoder.ReadUint32(&flags).ok()); + EXPECT_EQ(expected_metadata.flags(), flags); + } + + int64_t timestamp; + EXPECT_TRUE(entry_decoder.Next().ok()); // timestamp + EXPECT_EQ(4U, entry_decoder.FieldNumber()); + EXPECT_TRUE(entry_decoder.ReadInt64(×tamp).ok()); + EXPECT_EQ(expected_timestamp, timestamp); +} + +TEST(UtilsTest, EncodeTokenizedLog) { + constexpr std::byte kTokenizedData[1] = {(std::byte)0x01}; + constexpr int64_t kExpectedTimestamp = 1; + std::byte encode_buffer[32]; + + pw::log_tokenized::Metadata metadata = + pw::log_tokenized::Metadata::Set<1, 2, 3, 4>(); + + Result<ConstByteSpan> result = EncodeTokenizedLog( + metadata, kTokenizedData, kExpectedTimestamp, encode_buffer); + EXPECT_TRUE(result.ok()); + + pw::protobuf::Decoder log_decoder(result.value()); + VerifyLogEntry(log_decoder, metadata, kTokenizedData, kExpectedTimestamp); + + result = EncodeTokenizedLog(metadata, + reinterpret_cast<const uint8_t*>(kTokenizedData), + sizeof(kTokenizedData), + kExpectedTimestamp, + encode_buffer); + EXPECT_TRUE(result.ok()); + + log_decoder.Reset(result.value()); + VerifyLogEntry(log_decoder, metadata, kTokenizedData, kExpectedTimestamp); +} + +TEST(UtilsTest, EncodeTokenizedLog_EmptyFlags) { + constexpr std::byte kTokenizedData[1] = {(std::byte)0x01}; + constexpr int64_t kExpectedTimestamp = 1; + std::byte encode_buffer[32]; + + // Create an empty flags set. + pw::log_tokenized::Metadata metadata = + pw::log_tokenized::Metadata::Set<1, 2, 0, 4>(); + + Result<ConstByteSpan> result = EncodeTokenizedLog( + metadata, kTokenizedData, kExpectedTimestamp, encode_buffer); + EXPECT_TRUE(result.ok()); + + pw::protobuf::Decoder log_decoder(result.value()); + VerifyLogEntry(log_decoder, metadata, kTokenizedData, kExpectedTimestamp); +} + +TEST(UtilsTest, EncodeTokenizedLog_InsufficientSpace) { + constexpr std::byte kTokenizedData[1] = {(std::byte)0x01}; + constexpr int64_t kExpectedTimestamp = 1; + std::byte encode_buffer[1]; + + pw::log_tokenized::Metadata metadata = + pw::log_tokenized::Metadata::Set<1, 2, 3, 4>(); + + Result<ConstByteSpan> result = EncodeTokenizedLog( + metadata, kTokenizedData, kExpectedTimestamp, encode_buffer); + EXPECT_TRUE(result.status().IsResourceExhausted()); +} + +} // namespace pw::log
diff --git a/pw_log/protobuf.rst b/pw_log/protobuf.rst index 2ee7ea3..12c1f62 100644 --- a/pw_log/protobuf.rst +++ b/pw_log/protobuf.rst
@@ -50,3 +50,30 @@ and detokenize tokenized fields as appropriate. See :ref:`module-pw_tokenizer-proto` for details. + +Utility functions +----------------- +Conversion into the ``log.proto`` format from a tokenized log message can be +performed using the ``pw_log/proto_utils.h`` headers. Given tokenized data and +a payload, the headers provide a quick way to encode to the LogEntry protobuf. + +.. code-block:: cpp + + #include "pw_log/proto_utils.h" + + extern "C" void pw_tokenizer_HandleEncodedMessageWithPayload( + pw_tokenizer_Payload payload, const uint8_t data[], size_t size) { + pw::log_tokenized::Metadata metadata(payload); + std::byte log_buffer[kLogBufferSize]; + + Result<ConstByteSpan> result = EncodeTokenizedLog( + metadata, + std::as_bytes(std::span(data, size)), + log_buffer); + if (result.ok()) { + // This makes use of the encoded log proto and is custom per-product. + // It should be implemented by the caller and is not in Pigweed. + EmitProtoLogEntry(result.value()); + } + } +
diff --git a/pw_log/public/pw_log/proto_utils.h b/pw_log/public/pw_log/proto_utils.h new file mode 100644 index 0000000..fb1b6c3 --- /dev/null +++ b/pw_log/public/pw_log/proto_utils.h
@@ -0,0 +1,47 @@ +// Copyright 2021 The Pigweed Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not +// use this file except in compliance with the License. You may obtain a copy of +// the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations under +// the License. +#pragma once + +#include "pw_bytes/span.h" +#include "pw_log_tokenized/metadata.h" +#include "pw_result/result.h" + +namespace pw::log { + +// Convenience functions to convert from tokenized metadata to the log proto +// format. +// +// Return values: +// Ok - A byte span containing the encoded log proto. +// ResourceExhausted - The provided buffer was not large enough to store the +// proto. +Result<ConstByteSpan> EncodeTokenizedLog(log_tokenized::Metadata metadata, + ConstByteSpan tokenized_data, + int64_t ticks_since_epoch, + ByteSpan encode_buffer); + +inline Result<ConstByteSpan> EncodeTokenizedLog( + log_tokenized::Metadata metadata, + const uint8_t* tokenized_data, + size_t tokenized_data_size, + int64_t ticks_since_epoch, + ByteSpan encode_buffer) { + return EncodeTokenizedLog( + metadata, + std::as_bytes(std::span(tokenized_data, tokenized_data_size)), + ticks_since_epoch, + encode_buffer); +} + +} // namespace pw::log