Add support for checking output in benchmark
- Rename `test_data` to `test_data_input`.
- Add `test_data_output`, which contains golden output. The output
tensor of the benchmarked model will be validated against this.
Change-Id: I4c6e8111fcfe6a236d1876b5a8ae6b0ee859839f
diff --git a/benchmarks/benchmark.h b/benchmarks/benchmark.h
index e36020c..59383fb 100644
--- a/benchmarks/benchmark.h
+++ b/benchmarks/benchmark.h
@@ -21,6 +21,7 @@
uint32_t return_code;
uint32_t iterations;
uint64_t cycles;
+ uint32_t mismatch_count;
} BenchmarkOutputHeader;
#endif // #ifndef BENCHMARKS_BENCHMARK_H_
diff --git a/benchmarks/benchmark_kelvin.cc b/benchmarks/benchmark_kelvin.cc
index 1871619..7756b34 100644
--- a/benchmarks/benchmark_kelvin.cc
+++ b/benchmarks/benchmark_kelvin.cc
@@ -39,10 +39,16 @@
#define MODEL_HEADER STR(MODEL_HEADER_DIRECTORY BENCHMARK_NAME MODEL_HEADER_TYPE)
#include MODEL_HEADER
-#if (TEST_DATA == 1)
-#define TEST_DATA_HEADER_TYPE _test_data.h
-#define TEST_DATA_HEADER STR(MODEL_HEADER_DIRECTORY BENCHMARK_NAME TEST_DATA_HEADER_TYPE)
-#include TEST_DATA_HEADER
+#if (TEST_DATA_INPUT == 1)
+#define TEST_DATA_INPUT_HEADER_TYPE _input.h
+#define TEST_DATA_INPUT_HEADER STR(MODEL_HEADER_DIRECTORY BENCHMARK_NAME TEST_DATA_INPUT_HEADER_TYPE)
+#include TEST_DATA_INPUT_HEADER
+#endif
+
+#if (TEST_DATA_OUTPUT == 1)
+#define TEST_DATA_OUTPUT_HEADER_TYPE _output.h
+#define TEST_DATA_OUTPUT_HEADER STR(MODEL_HEADER_DIRECTORY BENCHMARK_NAME TEST_DATA_OUTPUT_HEADER_TYPE)
+#include TEST_DATA_OUTPUT_HEADER
#endif
namespace {
@@ -53,6 +59,7 @@
.return_code = 0, // Set by kelvin_start based on return value in main.
.iterations = 0,
.cycles = 0,
+ .mismatch_count = 0,
};
// This includes all ops currently used in the Kelvin model suite. More can be added.
@@ -127,8 +134,8 @@
}
TfLiteTensor* input = interpreter->input(0);
-#if (TEST_DATA == 1)
- memcpy(tflite::GetTensorData<uint8_t>(input), g_benchmark_test_data, input->bytes);
+#if (TEST_DATA_INPUT == 1)
+ memcpy(tflite::GetTensorData<uint8_t>(input), g_benchmark_input, input->bytes);
#else
memset(tflite::GetTensorData<uint8_t>(input), 0, input->bytes);
#endif
@@ -142,8 +149,8 @@
// TODO(michaelbrooks): Possibly set/verify test data?
for (int i = 0; i < iterations; ++i) {
-#if (TEST_DATA == 1)
- memcpy(tflite::GetTensorData<uint8_t>(input), g_benchmark_test_data, input->bytes);
+#if (TEST_DATA_INPUT == 1)
+ memcpy(tflite::GetTensorData<uint8_t>(input), g_benchmark_input, input->bytes);
#else
memset(tflite::GetTensorData<uint8_t>(input), 0, input->bytes);
#endif
@@ -152,6 +159,20 @@
uint64_t end = mcycle_read();
uint64_t num_cycles = end - begin;
+#if (TEST_DATA_OUTPUT == 1)
+ TfLiteTensor* output = interpreter->output(0);
+ int mismatch_count = 0;
+ for (size_t i = 0; i < output->bytes; ++i) {
+ int8_t vx = (int8_t)(*(tflite::GetTensorData<int8_t>(output) + i) & 0xFF);
+ int8_t vy = (int8_t)(*(g_benchmark_output + i) & 0xFF);
+ auto delta = ((vx) > (vy)) ? ((vx) - (vy)) : ((vy) - (vx));
+ if (delta) {
+ mismatch_count += 1;
+ }
+ }
+ output_header.mismatch_count = mismatch_count;
+#endif
+
#if (PROFILE == 1)
profiler.LogCsv();
#endif
@@ -165,6 +186,9 @@
LOG_INFO("Iterations: %ld", output_header.iterations);
_print64("Total Cycles: ", output_header.cycles);
_print64("Average Cycles per Iteration: ", average_cycles);
+#if (TEST_DATA_OUTPUT == 1)
+ LOG_INFO("Mismatch_count: %d", mismatch_count);
+#endif
LOG_INFO("========== End Benchmark ==========");
return kSuccess;
}
diff --git a/benchmarks/benchmark_smc.c b/benchmarks/benchmark_smc.c
index 2915dca..7edaf46 100644
--- a/benchmarks/benchmark_smc.c
+++ b/benchmarks/benchmark_smc.c
@@ -140,11 +140,13 @@
uint64_t average_cycles = udiv64_slow(cycles, iterations, NULL);
uint64_t wall_time_us = timer_finish - timer_start;
uint64_t average_wall_time_us = udiv64_slow(wall_time_us, iterations, NULL);
+ uint32_t mismatch_count = output_header_ptr->mismatch_count;
LOG_INFO("Iterations: %d", iterations);
_print64("Total Cycles", cycles);
_print64("Average Cycles per Iteration", average_cycles);
_print64("Wall time (us)", wall_time_us);
_print64("Wall time per Iteration (us)", average_wall_time_us);
+ LOG_INFO("Mismatch count: %d", mismatch_count);
LOG_INFO("========== End Benchmark ==========");
while (true) {
wait_for_interrupt();
diff --git a/benchmarks/benchmarks.bzl b/benchmarks/benchmarks.bzl
index 3414e7d..5f919f8 100644
--- a/benchmarks/benchmarks.bzl
+++ b/benchmarks/benchmarks.bzl
@@ -21,7 +21,8 @@
name,
model,
iterations,
- test_data = None,
+ test_data_input = None,
+ test_data_output = None,
profile = False,
kelvin_binary_info = None,
benchmark_path = "benchmarks",
@@ -51,14 +52,22 @@
)
kelvin_headers.append(model_header_name)
- if test_data:
- test_data_header_name = "{}_test_data".format(name)
+ if test_data_input:
+ input_header_name = "{}_input".format(name)
bin_to_c_file(
- name = test_data_header_name,
- srcs = [test_data],
- var_name = "g_benchmark_test_data",
+ name = input_header_name,
+ srcs = [test_data_input],
+ var_name = "g_benchmark_input",
)
- kelvin_headers.append(test_data_header_name)
+ kelvin_headers.append(input_header_name)
+ if test_data_output:
+ output_header_name = "{}_output".format(name)
+ bin_to_c_file(
+ name = output_header_name,
+ srcs = [test_data_output],
+ var_name = "g_benchmark_output",
+ )
+ kelvin_headers.append(output_header_name)
# Test to run in simulator and MPACT.
kelvin_test(
@@ -68,7 +77,8 @@
copts = [
"-DITERATIONS={}".format(iterations),
"-DBENCHMARK_NAME={}".format(name),
- "-DTEST_DATA={}".format(1 if test_data else 0),
+ "-DTEST_DATA_INPUT={}".format(1 if test_data_input else 0),
+ "-DTEST_DATA_OUTPUT={}".format(1 if test_data_output else 0),
"-DPROFILE={}".format(1 if profile else 0),
"-DBENCHMARK_PATH={}".format(benchmark_path),
],
@@ -175,7 +185,8 @@
model,
device_type,
iterations,
- test_data = None,
+ test_data_input = None,
+ test_data_output = None,
profile = False,
kelvin_binary_info = None,
benchmark_path = "benchmarks",
@@ -254,14 +265,22 @@
)
kelvin_headers.append(model_header_name)
- if test_data:
- test_data_header_name = "{}_test_data".format(name)
+ if test_data_input:
+ input_header_name = "{}_input".format(name)
bin_to_c_file(
- name = test_data_header_name,
- srcs = [test_data],
- var_name = "g_benchmark_test_data",
+ name = input_header_name,
+ srcs = [test_data_input],
+ var_name = "g_benchmark_input",
)
- kelvin_headers.append(test_data_header_name)
+ kelvin_headers.append(input_header_name)
+ if test_data_output:
+ output_header_name = "{}_output".format(name)
+ bin_to_c_file(
+ name = output_header_name,
+ srcs = [test_data_output],
+ var_name = "g_benchmark_output",
+ )
+ kelvin_headers.append(output_header_name)
kelvin_binary(
name = "{}_kelvin".format(name),
@@ -271,7 +290,8 @@
copts = [
"-DITERATIONS={}".format(iterations),
"-DBENCHMARK_NAME={}".format(name),
- "-DTEST_DATA={}".format(1 if test_data else 0),
+ "-DTEST_DATA_INPUT={}".format(1 if test_data_input else 0),
+ "-DTEST_DATA_OUTPUT={}".format(1 if test_data_output else 0),
"-DPROFILE={}".format(1 if profile else 0),
"-DBENCHMARK_PATH={}".format(benchmark_path),
],
