Energy op (#2121) `port c++ energy op to open source in tflm_signal` -port energy op and corresponding to new open source location for C++ BUG=[b/289422411](https://b.corp.google.com/issues/289422411)
diff --git a/python/tflite_micro/python_ops_resolver.cc b/python/tflite_micro/python_ops_resolver.cc index f67f7dd..4aad245 100644 --- a/python/tflite_micro/python_ops_resolver.cc +++ b/python/tflite_micro/python_ops_resolver.cc
@@ -45,6 +45,7 @@ AddDequantize(); AddDetectionPostprocess(); AddDiv(); + AddEnergy(); AddElu(); AddEqual(); AddEthosU();
diff --git a/signal/micro/kernels/BUILD b/signal/micro/kernels/BUILD index 6357db5..4b01baa 100644 --- a/signal/micro/kernels/BUILD +++ b/signal/micro/kernels/BUILD
@@ -9,6 +9,7 @@ name = "register_signal_ops", srcs = [ "delay.cc", + "energy.cc", "framer.cc", "overlap_add.cc", "rfft.cc", @@ -24,6 +25,7 @@ ], deps = [ "//signal/src:circular_buffer", + "//signal/src:energy", "//signal/src:overlap_add", "//signal/src:rfft", "//signal/src:window", @@ -194,3 +196,29 @@ "//tensorflow/lite/micro/testing:micro_test", ], ) + +cc_library( + name = "energy_flexbuffers_generated_data", + srcs = [ + "energy_flexbuffers_generated_data.cc", + ], + hdrs = [ + "energy_flexbuffers_generated_data.h", + ], +) + +cc_test( + name = "energy_test", + srcs = [ + "energy_test.cc", + ], + deps = [ + ":energy_flexbuffers_generated_data", + ":register_signal_ops", + "//tensorflow/lite/c:common", + "//tensorflow/lite/micro:op_resolvers", + "//tensorflow/lite/micro:test_helpers", + "//tensorflow/lite/micro/kernels:kernel_runner", + "//tensorflow/lite/micro/testing:micro_test", + ], +)
diff --git a/signal/micro/kernels/energy.cc b/signal/micro/kernels/energy.cc new file mode 100644 index 0000000..fa79bc8 --- /dev/null +++ b/signal/micro/kernels/energy.cc
@@ -0,0 +1,112 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "signal/src/energy.h" + +#include <math.h> +#include <stddef.h> +#include <stdint.h> + +#include "tensorflow/lite/kernels/internal/tensor_ctypes.h" +#include "tensorflow/lite/kernels/kernel_util.h" +#include "tensorflow/lite/micro/flatbuffer_utils.h" +#include "tensorflow/lite/micro/kernels/kernel_util.h" +#include "tensorflow/lite/micro/micro_context.h" + +namespace tflite { +namespace { + +constexpr int kInputTensor = 0; +constexpr int kOutputTensor = 0; + +// Indices into the init flexbuffer's vector. +// The parameter's name is in the comment that follows. +// Elements in the vectors are ordered alphabetically by parameter name. +constexpr int kEndIndexIndex = 0; // 'end_index' +constexpr int kStartIndexIndex = 1; // 'start_index' + +struct TFLMSignalEnergyParams { + int32_t end_index; + int32_t start_index; +}; + +void* Init(TfLiteContext* context, const char* buffer, size_t length) { + TFLITE_DCHECK(context->AllocatePersistentBuffer != nullptr); + + auto* data = + static_cast<TFLMSignalEnergyParams*>(context->AllocatePersistentBuffer( + context, sizeof(TFLMSignalEnergyParams))); + + if (data == nullptr) { + return nullptr; + } + + tflite::FlexbufferWrapper fbw(reinterpret_cast<const uint8_t*>(buffer), + length); + data->end_index = fbw.ElementAsInt32(kEndIndexIndex); + data->start_index = fbw.ElementAsInt32(kStartIndexIndex); + return data; +} + +TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) { + TF_LITE_ENSURE_EQ(context, NumInputs(node), 1); + TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1); + + MicroContext* micro_context = GetMicroContext(context); + TfLiteTensor* input = + micro_context->AllocateTempInputTensor(node, kInputTensor); + TF_LITE_ENSURE(context, input != nullptr); + TfLiteTensor* output = + micro_context->AllocateTempOutputTensor(node, kOutputTensor); + TF_LITE_ENSURE(context, output != nullptr); + + TF_LITE_ENSURE_EQ(context, NumDimensions(input), 1); + TF_LITE_ENSURE_EQ(context, NumDimensions(output), 1); + + TF_LITE_ENSURE_TYPES_EQ(context, input->type, kTfLiteInt16); + TF_LITE_ENSURE_TYPES_EQ(context, output->type, kTfLiteUInt32); + + micro_context->DeallocateTempTfLiteTensor(input); + micro_context->DeallocateTempTfLiteTensor(output); + return kTfLiteOk; +} + +TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) { + auto* params = reinterpret_cast<TFLMSignalEnergyParams*>(node->user_data); + + const TfLiteEvalTensor* input = + tflite::micro::GetEvalInput(context, node, kInputTensor); + TfLiteEvalTensor* output = + tflite::micro::GetEvalOutput(context, node, kOutputTensor); + + const Complex<int16_t>* input_data = + tflite::micro::GetTensorData<Complex<int16_t>>(input); + uint32_t* output_data = tflite::micro::GetTensorData<uint32_t>(output); + + tflm_signal::SpectrumToEnergy(input_data, params->start_index, + params->end_index, output_data); + return kTfLiteOk; +} + +} // namespace + +namespace tflm_signal { +TFLMRegistration* Register_ENERGY() { + static TFLMRegistration r = tflite::micro::RegisterOp(Init, Prepare, Eval); + return &r; +} +} // namespace tflm_signal + +} // namespace tflite
diff --git a/signal/micro/kernels/energy_flexbuffers_generated_data.cc b/signal/micro/kernels/energy_flexbuffers_generated_data.cc new file mode 100644 index 0000000..e25a577 --- /dev/null +++ b/signal/micro/kernels/energy_flexbuffers_generated_data.cc
@@ -0,0 +1,37 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// This file is generated. See: +// tensorflow/lite/micro/kernels/test_data_generation/README.md + +#include "signal/micro/kernels/energy_flexbuffers_generated_data.h" + +const int g_gen_data_size_start_index_2_end_index_4 = 35; +const unsigned char g_gen_data_start_index_2_end_index_4[] = { + 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, + 0x65, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, 0x02, 0x0b, + 0x18, 0x02, 0x01, 0x02, 0x04, 0x02, 0x04, 0x04, 0x04, 0x24, 0x01, +}; +const int g_gen_data_size_start_index_0_end_index_4 = 35; +const unsigned char g_gen_data_start_index_0_end_index_4[] = { + 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, + 0x65, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, 0x02, 0x0b, + 0x18, 0x02, 0x01, 0x02, 0x04, 0x00, 0x04, 0x04, 0x04, 0x24, 0x01, +}; +const int g_gen_data_size_start_index_4_end_index_8 = 35; +const unsigned char g_gen_data_start_index_4_end_index_8[] = { + 0x73, 0x74, 0x61, 0x72, 0x74, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, + 0x65, 0x6e, 0x64, 0x5f, 0x69, 0x6e, 0x64, 0x65, 0x78, 0x00, 0x02, 0x0b, + 0x18, 0x02, 0x01, 0x02, 0x08, 0x04, 0x04, 0x04, 0x04, 0x24, 0x01, +};
diff --git a/signal/micro/kernels/energy_flexbuffers_generated_data.h b/signal/micro/kernels/energy_flexbuffers_generated_data.h new file mode 100644 index 0000000..f2840f6 --- /dev/null +++ b/signal/micro/kernels/energy_flexbuffers_generated_data.h
@@ -0,0 +1,28 @@ +/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef SIGNAL_MICRO_KERNELS_TEST_DATA_GENERATION_GENERATE_ENERGY_FLEXBUFFERS_DATA_H_ +#define SIGNAL_MICRO_KERNELS_TEST_DATA_GENERATION_GENERATE_ENERGY_FLEXBUFFERS_DATA_H_ + +extern const int g_gen_data_size_start_index_2_end_index_4; +extern const unsigned char g_gen_data_start_index_2_end_index_4[]; + +extern const int g_gen_data_size_start_index_0_end_index_4; +extern const unsigned char g_gen_data_start_index_0_end_index_4[]; + +extern const int g_gen_data_size_start_index_4_end_index_8; +extern const unsigned char g_gen_data_start_index_4_end_index_8[]; + +#endif // SIGNAL_MICRO_KERNELS_TEST_DATA_GENERATION_GENERATE_ENERGY_FLEXBUFFERS_DATA_H_
diff --git a/signal/micro/kernels/energy_test.cc b/signal/micro/kernels/energy_test.cc new file mode 100644 index 0000000..64f0619 --- /dev/null +++ b/signal/micro/kernels/energy_test.cc
@@ -0,0 +1,123 @@ +/* Copyright 2021 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "signal/micro/kernels/energy_flexbuffers_generated_data.h" +#include "tensorflow/lite/micro/kernels/kernel_runner.h" +#include "tensorflow/lite/micro/test_helpers.h" +#include "tensorflow/lite/micro/testing/micro_test.h" + +namespace tflite { +namespace testing { +namespace { + +TfLiteStatus TestEnergy(int* input_dims_data, const int16_t* input_data, + int* output_dims_data, const uint32_t* golden, + const unsigned char* flexbuffers_data, + const unsigned int flexbuffers_data_size, + uint32_t* output_data) { + TfLiteIntArray* input_dims = IntArrayFromInts(input_dims_data); + TfLiteIntArray* output_dims = IntArrayFromInts(output_dims_data); + const int output_len = ElementCount(*output_dims); + constexpr int kInputsSize = 1; + constexpr int kOutputsSize = 1; + constexpr int kTensorsSize = kInputsSize + kOutputsSize; + TfLiteTensor tensors[kTensorsSize] = { + CreateTensor(input_data, input_dims), + CreateTensor(output_data, output_dims), + }; + + int inputs_array_data[] = {1, 0}; + TfLiteIntArray* inputs_array = IntArrayFromInts(inputs_array_data); + int outputs_array_data[] = {1, 1}; + TfLiteIntArray* outputs_array = IntArrayFromInts(outputs_array_data); + + const TFLMRegistration* registration = tflm_signal::Register_ENERGY(); + micro::KernelRunner runner(*registration, tensors, kTensorsSize, inputs_array, + outputs_array, + /*builtin_data=*/nullptr); + + // TfLite uses a char* for the raw bytes whereas flexbuffers use an unsigned + // char*. This small discrepancy results in compiler warnings unless we + // reinterpret_cast right before passing in the flexbuffer bytes to the + // KernelRunner. + TfLiteStatus status = runner.InitAndPrepare( + reinterpret_cast<const char*>(flexbuffers_data), flexbuffers_data_size); + if (status != kTfLiteOk) { + return status; + } + + status = runner.Invoke(); + if (status != kTfLiteOk) { + return status; + } + + for (int i = 0; i < output_len; ++i) { + TF_LITE_MICRO_EXPECT_EQ(golden[i], output_data[i]); + } + return kTfLiteOk; +} + +} // namespace +} // namespace testing +} // namespace tflite + +TF_LITE_MICRO_TESTS_BEGIN + +TF_LITE_MICRO_TEST(EnergyTestMiddle) { + int input_shape[] = {1, 16}; + int output_shape[] = {1, 8}; + const int16_t input[] = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16}; + const uint32_t golden[] = {0, 0, 61, 113, 0, 0, 0, 0}; + uint32_t output[8]; + memset(output, 0, sizeof(output)); + TF_LITE_MICRO_EXPECT_EQ( + kTfLiteOk, tflite::testing::TestEnergy( + input_shape, input, output_shape, golden, + g_gen_data_start_index_2_end_index_4, + g_gen_data_size_start_index_2_end_index_4, output)); +} + +TF_LITE_MICRO_TEST(EnergyTestStart) { + int input_shape[] = {1, 16}; + int output_shape[] = {1, 8}; + const int16_t input[] = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16}; + const uint32_t golden[] = {5, 25, 61, 113, 0, 0, 0, 0}; + uint32_t output[8]; + memset(output, 0, sizeof(output)); + TF_LITE_MICRO_EXPECT_EQ( + kTfLiteOk, tflite::testing::TestEnergy( + input_shape, input, output_shape, golden, + g_gen_data_start_index_0_end_index_4, + g_gen_data_size_start_index_0_end_index_4, output)); +} + +TF_LITE_MICRO_TEST(EnergyTestEnd) { + int input_shape[] = {1, 16}; + int output_shape[] = {1, 8}; + const int16_t input[] = {1, 2, 3, 4, 5, 6, 7, 8, + 9, 10, 11, 12, 13, 14, 15, 16}; + const uint32_t golden[] = {0, 0, 0, 0, 181, 265, 365, 481}; + uint32_t output[8]; + memset(output, 0, sizeof(output)); + TF_LITE_MICRO_EXPECT_EQ( + kTfLiteOk, tflite::testing::TestEnergy( + input_shape, input, output_shape, golden, + g_gen_data_start_index_4_end_index_8, + g_gen_data_size_start_index_4_end_index_8, output)); +} + +TF_LITE_MICRO_TESTS_END
diff --git a/signal/src/BUILD b/signal/src/BUILD index ecb712b..638adcf 100644 --- a/signal/src/BUILD +++ b/signal/src/BUILD
@@ -39,3 +39,10 @@ srcs = ["overlap_add.cc"], hdrs = ["overlap_add.h"], ) + +cc_library( + name = "energy", + srcs = ["energy.cc"], + hdrs = ["energy.h"], + deps = [":complex"], +)
diff --git a/signal/src/energy.cc b/signal/src/energy.cc new file mode 100644 index 0000000..3ea5fc0 --- /dev/null +++ b/signal/src/energy.cc
@@ -0,0 +1,33 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "signal/src/energy.h" + +#include "signal/src/complex.h" + +namespace tflite { +namespace tflm_signal { +void SpectrumToEnergy(const Complex<int16_t>* input, int start_index, + int end_index, uint32_t* output) { + for (int i = start_index; i < end_index; i++) { + const int16_t real = input[i].real; // 15 bits + const int16_t imag = input[i].imag; // 15 bits + // 31 bits + output[i] = (static_cast<int32_t>(real) * real) + + (static_cast<int32_t>(imag) * imag); + } +} + +} // namespace tflm_signal +} // namespace tflite
diff --git a/signal/src/energy.h b/signal/src/energy.h new file mode 100644 index 0000000..5a1cf37 --- /dev/null +++ b/signal/src/energy.h
@@ -0,0 +1,38 @@ +/* Copyright 2019 The TensorFlow Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef SIGNAL_ENERGY_H_ +#define SIGNAL_ENERGY_H_ + +#include <stdint.h> + +#include "signal/src/complex.h" + +namespace tflite { +namespace tflm_signal { +// TODO(b/286250473): remove namespace once de-duped libraries above + +// Calculates the power spectrum from a DFT output between start and end indices +// +// * `start_index` and `end_index` must valid indices into `input` +// * `output` must be the same size as `input`. Only the values at indices +// `start_index` and `end_index` inclusive should be considered valid. +void SpectrumToEnergy(const Complex<int16_t>* input, int start_index, + int end_index, uint32_t* output); + +} // namespace tflm_signal +} // namespace tflite + +#endif // SIGNAL_ENERGY_H_
diff --git a/tensorflow/lite/micro/kernels/Makefile.inc b/tensorflow/lite/micro/kernels/Makefile.inc index 8fbbb34..e64c0f4 100644 --- a/tensorflow/lite/micro/kernels/Makefile.inc +++ b/tensorflow/lite/micro/kernels/Makefile.inc
@@ -53,6 +53,11 @@ $(TENSORFLOW_ROOT)signal/micro/kernels/delay_flexbuffers_generated_data.cc, \ $(TENSORFLOW_ROOT)signal/micro/kernels/delay_flexbuffers_generated_data.h)) +$(eval $(call microlite_test,kernel_signal_energy_test,\ + $(TENSORFLOW_ROOT)signal/micro/kernels/energy_test.cc \ + $(TENSORFLOW_ROOT)signal/micro/kernels/energy_flexbuffers_generated_data.cc, \ + $(TENSORFLOW_ROOT)signal/micro/kernels/energy_flexbuffers_generated_data.h)) + $(eval $(call microlite_test,kernel_signal_fft_test,\ $(TENSORFLOW_ROOT)signal/micro/kernels/fft_test.cc \ $(TENSORFLOW_ROOT)signal/micro/kernels/fft_flexbuffers_generated_data.cc \
diff --git a/tensorflow/lite/micro/kernels/micro_ops.h b/tensorflow/lite/micro/kernels/micro_ops.h index 3482231..e14d327 100644 --- a/tensorflow/lite/micro/kernels/micro_ops.h +++ b/tensorflow/lite/micro/kernels/micro_ops.h
@@ -135,6 +135,7 @@ // TODO(b/160234179): Change custom OPs to also return by value. namespace tflm_signal { TFLMRegistration* Register_DELAY(); +TFLMRegistration* Register_ENERGY(); TFLMRegistration* Register_FRAMER(); TFLMRegistration* Register_OVERLAP_ADD(); TFLMRegistration* Register_STACKER();
diff --git a/tensorflow/lite/micro/micro_mutable_op_resolver.h b/tensorflow/lite/micro/micro_mutable_op_resolver.h index df80079..09cd25d 100644 --- a/tensorflow/lite/micro/micro_mutable_op_resolver.h +++ b/tensorflow/lite/micro/micro_mutable_op_resolver.h
@@ -224,6 +224,11 @@ return AddBuiltin(BuiltinOperator_DIV, tflite::Register_DIV(), ParseDiv); } + TfLiteStatus AddEnergy() { + // TODO(b/286250473): change back name to "Energy" and remove namespace + return AddCustom("SignalEnergy", tflite::tflm_signal::Register_ENERGY()); + } + TfLiteStatus AddElu() { return AddBuiltin(BuiltinOperator_ELU, tflite::Register_ELU(), ParseElu); }
diff --git a/tensorflow/lite/micro/tools/make/Makefile b/tensorflow/lite/micro/tools/make/Makefile index e002cf1..f432f1b 100644 --- a/tensorflow/lite/micro/tools/make/Makefile +++ b/tensorflow/lite/micro/tools/make/Makefile
@@ -313,12 +313,14 @@ MICROLITE_CC_KERNEL_SRCS := \ $(TENSORFLOW_ROOT)signal/micro/kernels/delay.cc \ +$(TENSORFLOW_ROOT)signal/micro/kernels/energy.cc \ $(TENSORFLOW_ROOT)signal/micro/kernels/framer.cc \ $(TENSORFLOW_ROOT)signal/micro/kernels/rfft.cc \ $(TENSORFLOW_ROOT)signal/micro/kernels/stacker.cc \ $(TENSORFLOW_ROOT)signal/micro/kernels/overlap_add.cc \ $(TENSORFLOW_ROOT)signal/micro/kernels/window.cc \ $(TENSORFLOW_ROOT)signal/src/circular_buffer.cc \ +$(TENSORFLOW_ROOT)signal/src/energy.cc \ $(TENSORFLOW_ROOT)signal/src/overlap_add.cc \ $(TENSORFLOW_ROOT)signal/src/rfft_float.cc \ $(TENSORFLOW_ROOT)signal/src/rfft_int16.cc \